Coverage for jutil/format.py: 82%

# Coverage for jutil/format.py : 82%

## 268 statements

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1 import csv

2 import html

3 import json

4 import logging

5 import os

6 import re

7 import tempfile

8 from collections import OrderedDict

9 from datetime import timedelta

10 from decimal import Decimal

11 import subprocess

12 from io import StringIO

13 from typing import List, Any, Optional, Union, Dict, Sequence, Tuple, TypeVar

14 from django.conf import settings

15 from django.core.exceptions import ValidationError

16 from django.utils.functional import lazy

17 import xml.dom.minidom # type: ignore

18 from django.utils.safestring import mark_safe

19 from django.utils.text import capfirst

20

21 logger = logging.getLogger(__name__)

22

23 S = TypeVar('S')

24

25

26 def format_full_name(first_name: str, last_name: str, max_length: int = 20) -> str:

27 """

28 Limits name length to specified length. Tries to keep name as human-readable an natural as possible.

29 :param first_name: First name

30 :param last_name: Last name

31 :param max_length: Maximum length

32 :return: Full name of shortened version depending on length

33 """

34 # dont allow commas in limited names

35 first_name = first_name.replace(',', ' ')

36 last_name = last_name.replace(',', ' ')

37

38 # accept short full names as is

39 original_full_name = first_name + ' ' + last_name

40 if len(original_full_name) <= max_length:

41 return original_full_name

42

43 # drop middle names

44 first_name = first_name.split(' ')[0]

45 full_name = first_name + ' ' + last_name

46 if len(full_name) <= max_length:

47 return full_name

48

49 # drop latter parts of combined first names

50 first_name = re.split(r'[\s\-]', first_name)[0]

51 full_name = first_name + ' ' + last_name

52 if len(full_name) <= max_length:

53 return full_name

54

55 # drop latter parts of multi part last names

56 last_name = re.split(r'[\s\-]', last_name)[0]

57 full_name = first_name + ' ' + last_name

58 if len(full_name) <= max_length:

59 return full_name

60

61 # shorten last name to one letter

62 last_name = last_name[:1]

63

64 full_name = first_name + ' ' + last_name

65 if len(full_name) > max_length:

66 raise Exception('Failed to shorten name {}'.format(original_full_name))

67 return full_name

68

69

70 def format_timedelta(dt: timedelta, days_label: str = 'd', hours_label: str = 'h',

71 minutes_label: str = 'min', seconds_label: str = 's') -> str:

72 """

73 Formats timedelta to readable format, e.g. 1h30min15s.

74 :param dt: timedelta

75 :param days_label: Label for days. Leave empty '' if value should be skipped / ignored.

76 :param hours_label: Label for hours. Leave empty '' if value should be skipped / ignored.

77 :param minutes_label: Label for minutes. Leave empty '' if value should be skipped / ignored.

78 :param seconds_label: Label for seconds. Leave empty '' if value should be skipped / ignored.

79 :return: str

80 """

81 parts = (

82 (86400, days_label),

83 (3600, hours_label),

84 (60, minutes_label),

85 (1, seconds_label),

86 )

87 out = ""

88 seconds_f = dt.total_seconds()

89 seconds = int(seconds_f)

90 for n_secs, label in parts:

91 n, remainder = divmod(seconds, n_secs)

92 if n > 0 and label:

93 out += str(n) + label

94 seconds = remainder

95 out_str = out.strip()

96 if not out_str:

97 if seconds_f >= 0.001: 97 ↛ 100line 97 didn't jump to line 100, because the condition on line 97 was never false

98 out_str = '{:0.3f}'.format(int(seconds_f * 1000.0) * 0.001) + seconds_label

99 else:

100 out_str = '0' + seconds_label

101 return out_str.strip()

102

103

104 def format_xml(content: str, encoding: str = 'UTF-8', exceptions: bool = False) -> str:

105 """

106 Formats XML document as human-readable plain text.

107 If settings.XMLLINT_PATH is defined xmllint is used for formatting (higher quality). Otherwise minidom toprettyxml is used.

108 :param content: XML data as str

109 :param encoding: XML file encoding

110 :param exceptions: Raise exceptions on error

111 :return: str (Formatted XML str)

112 """

113 assert isinstance(content, str)

114 try:

115 if hasattr(settings, 'XMLLINT_PATH') and settings.XMLLINT_PATH: 115 ↛ 121line 115 didn't jump to line 121, because the condition on line 115 was never false

116 with tempfile.NamedTemporaryFile() as fp:

117 fp.write(content.encode(encoding=encoding))

118 fp.flush()

119 out = subprocess.check_output([settings.XMLLINT_PATH, '--format', fp.name])

120 return out.decode(encoding=encoding)

121 return xml.dom.minidom.parseString(content).toprettyxml()

122 except Exception as e:

123 logger.error('format_xml failed: %s', e)

124 if exceptions:

125 raise

126 return content

127

128

129 def format_xml_bytes(content: bytes, encoding: str = 'UTF-8', exceptions: bool = False) -> bytes:

130 """

131 Formats XML document as human-readable plain text and returns result in bytes.

132 If settings.XMLLINT_PATH is defined xmllint is used for formatting (higher quality). Otherwise minidom toprettyxml is used.

133 :param content: XML data as bytes

134 :param encoding: XML file encoding

135 :param exceptions: Raise exceptions on error

136 :return: bytes (Formatted XML as bytes)

137 """

138 assert isinstance(content, bytes)

139 try:

140 if hasattr(settings, 'XMLLINT_PATH') and settings.XMLLINT_PATH: 140 ↛ 146line 140 didn't jump to line 146, because the condition on line 140 was never false

141 with tempfile.NamedTemporaryFile() as fp:

142 fp.write(content)

143 fp.flush()

144 out = subprocess.check_output([settings.XMLLINT_PATH, '--format', fp.name])

145 return out

146 return xml.dom.minidom.parseString(content.decode(encoding=encoding)).toprettyxml(encoding=encoding)

147 except Exception as e:

148 logger.error('format_xml_bytes failed: %s', e)

149 if exceptions:

150 raise

151 return content

152

153

154 def format_xml_file(full_path: str, encoding: str = 'UTF-8', exceptions: bool = False) -> bytes:

155 """

156 Formats XML file as human-readable plain text and returns result in bytes.

157 Tries to format XML file first, if formatting fails the file content is returned as is.

158 If the file does not exist empty bytes is returned.

159 If settings.XMLLINT_PATH is defined xmllint is used for formatting (higher quality). Otherwise minidom toprettyxml is used.

160 :param full_path: Full path to XML file

161 :param encoding: XML file encoding

162 :param exceptions: Raise exceptions on error

163 :return: bytes

164 """

165 try:

166 if hasattr(settings, 'XMLLINT_PATH') and settings.XMLLINT_PATH:

167 return subprocess.check_output([settings.XMLLINT_PATH, '--format', full_path])

168 with open(full_path, 'rb') as fp:

169 return xml.dom.minidom.parse(fp).toprettyxml(encoding=encoding)

170 except Exception as e:

171 logger.error('format_xml_file failed (1): %s', e)

172 if exceptions:

173 raise

174 try:

175 with open(full_path, 'rb') as fp:

176 return fp.read()

177 except Exception as e:

178 logger.error('format_xml_file failed (2): %s', e)

179 return b''

180

181

182 def format_as_html_json(value: Any) -> str:

183 """

184 Returns value as JSON-formatted value in HTML.

185 :param value: Any value which can be converted to JSON by json.dumps

186 :return: str

187 """

188 return mark_safe(html.escape(json.dumps(value, indent=4)).replace('\n', '<br/>').replace(' ', '&nbsp;'))

189

190

191 def _format_dict_as_html_key(k: str) -> str:

192 if k.startswith('@'):

193 k = k[1:]

194 k = k.replace('_', ' ')

195 k = re.sub(r'((?<=[a-z])[A-Z]|(?<!\A)[A-Z](?=[a-z]))', r' \1', k)

196 parts = k.split(' ')

197 out: List[str] = [str(capfirst(parts[0].strip()))]

198 for p in parts[1:]:

199 p2 = p.strip().lower()

200 if p2: 200 ↛ 198line 200 didn't jump to line 198, because the condition on line 200 was never false

201 out.append(p2)

202 return ' '.join(out)

203

204

205 def _format_dict_as_html_r(data: Dict[str, Any], margin: str = '', format_keys: bool = True) -> str:

206 if not isinstance(data, dict): 206 ↛ 207line 206 didn't jump to line 207, because the condition on line 206 was never true

207 return '{}{}\n'.format(margin, data)

208 out = ''

209 for k, v in OrderedDict(sorted(data.items())).items():

210 if isinstance(v, dict):

211 out += '{}{}:\n'.format(margin, _format_dict_as_html_key(k) if format_keys else k)

212 out += _format_dict_as_html_r(v, margin + ' ', format_keys=format_keys)

213 out += '\n'

214 elif isinstance(v, list): 214 ↛ 215line 214 didn't jump to line 215, because the condition on line 214 was never true

215 for v2 in v:

216 out += '{}{}:\n'.format(margin, _format_dict_as_html_key(k) if format_keys else k)

217 out += _format_dict_as_html_r(v2, margin + ' ', format_keys=format_keys)

218 out += '\n'

219 else:

220 out += '{}{}: {}\n'.format(margin, _format_dict_as_html_key(k) if format_keys else k, v)

221 return out

222

223

224 def format_dict_as_html(data: Dict[str, Any], format_keys: bool = True) -> str:

225 """

226 Formats dict to simple human readable pre-formatted html (<pre> tag).

227 :param data: dict

228 :param format_keys: Re-format 'additionalInfo' and 'additional_info' type of keys as 'Additional info'

229 :return: str (html)

230 """

231 return '<pre>' + _format_dict_as_html_r(data, format_keys=format_keys) + '</pre>'

232

233

234 def format_csv(rows: List[List[Any]], dialect: str = 'excel') -> str:

235 """

236 Formats rows to CSV string content.

237 :param rows: List[List[Any]]

238 :param dialect: See csv.writer dialect

239 :return: str

240 """

241 f = StringIO()

242 writer = csv.writer(f, dialect=dialect)

243 for row in rows:

244 writer.writerow(row)

245 return f.getvalue()

246

247

248 def format_table(rows: List[List[Any]], max_col: Optional[int] = None, max_line: Optional[int] = 200, # noqa

249 col_sep: str = '|', row_sep: str = '-', row_begin: str = '|', row_end: str = '|',

250 has_label_row: bool = False,

251 left_align: Optional[List[int]] = None, center_align: Optional[List[int]] = None) -> str:

252 """

253 Formats "ASCII-table" rows by padding column widths to longest column value, optionally limiting column widths.

254 Optionally separates colums with ' | ' character and header row with '-' characters.

255 Supports left, right and center alignment. Useful for console apps / debugging.

256

257 :param rows: List[List[Any]]

258 :param max_col: Max column value width. Pass None for unlimited length.

259 :param max_line: Maximum single line length. Exceeding columns truncated. Pass None for unlimited length.

260 :param col_sep: Column separator string.

261 :param row_sep: Row separator character used before first row, end, after first row (if has_label_row).

262 :param row_begin: Row begin string, inserted before each row.

263 :param row_end: Row end string, appended after each row.

264 :param has_label_row: Set to True if table starts with column label row.

265 :param left_align: Indexes of left-aligned columns. By default all are right aligned.

266 :param center_align: Indexes of center-aligned columns. By default all are right aligned.

267 :return: str

268 """

269 # validate parameters

270 assert max_col is None or max_col > 2

271 if left_align is None:

272 left_align = []

273 if center_align is None:

274 center_align = []

275 if left_align:

276 if set(left_align) & set(center_align): 276 ↛ 277line 276 didn't jump to line 277, because the condition on line 276 was never true

277 raise ValidationError('Left align columns {} overlap with center align {}'.format(left_align, center_align))

278

279 # find out number of columns

280 ncols = 0

281 for row in rows:

282 ncols = max(ncols, len(row))

283

284 # find out full-width column lengths

285 col_lens0: List[int] = [0] * ncols

286 for row in rows:

287 for ix, v in enumerate(row):

288 v = str(v)

289 col_lens0[ix] = max(col_lens0[ix], len(v))

290

291 # adjust max_col if needed

292 if max_line and (not max_col or sum(col_lens0) > max_line): 292 ↛ 293line 292 didn't jump to line 293, because the condition on line 292 was never true

293 max_col = max_line // ncols

294

295 # length limited lines and final column lengths

296 col_lens = [0] * ncols

297 lines: List[List[str]] = []

298 for row in rows:

299 line = []

300 for ix, v in enumerate(row):

301 v = str(v)

302 if max_col and len(v) > max_col:

303 v = v[:max_col-2] + '..'

304 line.append(v)

305 col_lens[ix] = max(col_lens[ix], len(v))

306 while len(line) < ncols: 306 ↛ 307line 306 didn't jump to line 307, because the condition on line 306 was never true

307 line.append('')

308 lines.append(line)

309

310 # padded lines

311 lines2: List[List[str]] = []

312 for line in lines:

313 line2 = []

314 for ix, v in enumerate(line):

315 col_len = col_lens[ix]

316 if len(v) < col_len:

317 if ix in left_align:

318 v = v + ' ' * (col_len - len(v))

319 elif ix in center_align:

320 pad = col_len - len(v)

321 lpad = int(pad/2)

322 rpad = pad - lpad

323 v = ' ' * lpad + v + ' '*rpad

324 else:

325 v = ' '*(col_len-len(v)) + v

326 line2.append(v)

327 lines2.append(line2)

328

329 # calculate max number of columns and max line length

330 max_line_len = 0

331 col_sep_len = len(col_sep)

332 ncols0 = ncols

333 for line in lines2:

334 if max_line is not None: 334 ↛ 333line 334 didn't jump to line 333, because the condition on line 334 was never false

335 line_len = len(row_begin) + sum(len(v)+col_sep_len for v in line[:ncols]) - col_sep_len + len(row_end)

336 while line_len > max_line:

337 ncols -= 1

338 line_len = len(row_begin) + sum(len(v)+col_sep_len for v in line[:ncols]) - col_sep_len + len(row_end)

339 max_line_len = max(max_line_len, line_len)

340

341 # find out how we should terminate lines/rows

342 line_term = ''

343 row_sep_term = ''

344 if ncols0 > ncols:

345 line_term = '..'

346 row_sep_term = row_sep * int(2 / len(row_sep))

347

348 # final output with row and column separators

349 lines3 = []

350 if row_sep: 350 ↛ 352line 350 didn't jump to line 352, because the condition on line 350 was never false

351 lines3.append(row_sep * max_line_len + row_sep_term)

352 for line_ix, line in enumerate(lines2):

353 while len(line) > ncols:

354 line.pop()

355 line_out = col_sep.join(line)

356 lines3.append(row_begin + line_out + row_end + line_term)

357 if line_ix == 0 and row_sep and has_label_row:

358 lines3.append(row_sep * max_line_len + row_sep_term)

359 if row_sep: 359 ↛ 361line 359 didn't jump to line 361, because the condition on line 359 was never false

360 lines3.append(row_sep * max_line_len + row_sep_term)

361 return '\n'.join(lines3)

362

363

364 def ucfirst(v: str) -> str:

365 """

366 Converts first character of the string to uppercase.

367 :param v: str

368 :return: str

369 """

370 return v[0:1].upper() + v[1:]

371

372

373 ucfirst_lazy = lazy(ucfirst, str)

374

375

376 def dec1(a: Union[float, int, Decimal, str]) -> Decimal:

377 """

378 Converts number to Decimal with 1 decimal digits.

379 :param a: Number

380 :return: Decimal with 1 decimal digits

381 """

382 return Decimal(a).quantize(Decimal('1.0'))

383

384

385 def dec2(a: Union[float, int, Decimal, str]) -> Decimal:

386 """

387 Converts number to Decimal with 2 decimal digits.

388 :param a: Number

389 :return: Decimal with 2 decimal digits

390 """

391 return Decimal(a).quantize(Decimal('1.00'))

392

393

394 def dec3(a: Union[float, int, Decimal, str]) -> Decimal:

395 """

396 Converts number to Decimal with 3 decimal digits.

397 :param a: Number

398 :return: Decimal with 3 decimal digits

399 """

400 return Decimal(a).quantize(Decimal('1.000'))

401

402

403 def dec4(a: Union[float, int, Decimal, str]) -> Decimal:

404 """

405 Converts number to Decimal with 4 decimal digits.

406 :param a: Number

407 :return: Decimal with 4 decimal digits

408 """

409 return Decimal(a).quantize(Decimal('1.0000'))

410

411

412 def dec5(a: Union[float, int, Decimal, str]) -> Decimal:

413 """

414 Converts number to Decimal with 5 decimal digits.

415 :param a: Number

416 :return: Decimal with 4 decimal digits

417 """

418 return Decimal(a).quantize(Decimal('1.00000'))

419

420

421 def dec6(a: Union[float, int, Decimal, str]) -> Decimal:

422 """

423 Converts number to Decimal with 6 decimal digits.

424 :param a: Number

425 :return: Decimal with 4 decimal digits

426 """

427 return Decimal(a).quantize(Decimal('1.000000'))

428

429

430 def is_media_full_path(file_path: str) -> bool:

431 """

432 Checks if file path is under (settings) MEDIA_ROOT.

433 """

434 return hasattr(settings, 'MEDIA_ROOT') and settings.MEDIA_ROOT and os.path.isabs(file_path) and \

435 os.path.realpath(file_path).startswith(settings.MEDIA_ROOT)

436

437

438 def strip_media_root(file_path: str) -> str:

439 """

440 If file path starts with (settings) MEDIA_ROOT,

441 the MEDIA_ROOT part gets stripped and only relative path is returned.

442 Otherwise file path is returned as is. This enabled stored file names in more

443 portable format for different environment / storage.

444 If MEDIA_ROOT is missing or empty, the filename is returned as is.

445 Reverse operation of this is get_media_full_path().

446 :param file_path: str

447 :return: str

448 """

449 full_path = os.path.realpath(file_path)

450 if not is_media_full_path(file_path): 450 ↛ 451line 450 didn't jump to line 451, because the condition on line 450 was never true

451 logger.error('strip_media_root() expects absolute path under MEDIA_ROOT, got %s (%s)', file_path, full_path)

452 raise ValueError('strip_media_root() expects absolute path under MEDIA_ROOT')

453 file_path = full_path[len(settings.MEDIA_ROOT):]

454 if file_path.startswith('/'): 454 ↛ 456line 454 didn't jump to line 456, because the condition on line 454 was never false

455 return file_path[1:]

456 return file_path

457

458

459 def get_media_full_path(file_path: str) -> str:

460 """

461 Returns the absolute path from a (relative) path to (settings) MEDIA_ROOT.

462 This enabled stored file names in more portable format for different environment / storage.

463 If MEDIA_ROOT is missing or non-media path is passed to function, exception is raised.

464 Reverse operation of this is strip_media_root().

465 :param file_path: str

466 :return: str

467 """

468 full_path = os.path.realpath(file_path) if os.path.isabs(file_path) else os.path.join(settings.MEDIA_ROOT, file_path)

469 if not is_media_full_path(full_path): 469 ↛ 470line 469 didn't jump to line 470, because the condition on line 469 was never true

470 logger.error('get_media_full_path() expects relative path to MEDIA_ROOT, got %s (%s)', file_path, full_path)

471 raise ValueError('get_media_full_path() expects relative path to MEDIA_ROOT')

472 return full_path

473

474

475 def camel_case_to_underscore(s: str) -> str:

476 """

477 Converts camelCaseWord to camel_case_word.

478 :param s: str

479 :return: str

480 """

481 if s: 481 ↛ 485line 481 didn't jump to line 485, because the condition on line 481 was never false

482 s = re.sub(r"([A-Z]+)([A-Z][a-z])", r'\1_\2', s)

483 s = re.sub(r"([a-z\d])([A-Z])", r'\1_\2', s)

484 s = s.replace("-", "_")

485 return s.lower()

486

487

488 def underscore_to_camel_case(s: str) -> str:

489 """

490 Converts under_score_word to underScoreWord.

491 :param s: str

492 :return: str

493 """

494 if s: 494 ↛ 497line 494 didn't jump to line 497, because the condition on line 494 was never false

495 p = s.split('_')

496 s = p[0] + ''.join([ucfirst(w) for w in p[1:]])

497 return s

498

499

500 def choices_label(choices: Sequence[Tuple[S, str]], value: S) -> str:

501 """

502 Iterates (value,label) list and returns label matching the choice

503 :param choices: [(choice1, label1), (choice2, label2), ...]

504 :param value: Value to find

505 :return: label or None

506 """

507 for key, label in choices: 507 ↛ 510line 507 didn't jump to line 510, because the loop on line 507 didn't complete

508 if key == value: 508 ↛ 507line 508 didn't jump to line 507, because the condition on line 508 was never false

509 return label

510 return ''