Coverage for jutil/format.py: 84%

# Coverage for jutil/format.py : 84%

## 212 statements

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1 import csv

2 import logging

3 import os

4 import re

5 import tempfile

6 from datetime import timedelta

7 from decimal import Decimal

8 import subprocess

9 from io import StringIO

10 from typing import List, Any, Optional, Union

11 from django.conf import settings

12 from django.core.exceptions import ValidationError

13 from django.utils.functional import lazy

14 import xml.dom.minidom # type: ignore

15

16

17 logger = logging.getLogger(__name__)

18

19

20 def format_full_name(first_name: str, last_name: str, max_length: int = 20) -> str:

21 """

22 Limits name length to specified length. Tries to keep name as human-readable an natural as possible.

23 :param first_name: First name

24 :param last_name: Last name

25 :param max_length: Maximum length

26 :return: Full name of shortened version depending on length

27 """

28 # dont allow commas in limited names

29 first_name = first_name.replace(',', ' ')

30 last_name = last_name.replace(',', ' ')

31

32 # accept short full names as is

33 original_full_name = first_name + ' ' + last_name

34 if len(original_full_name) <= max_length:

35 return original_full_name

36

37 # drop middle names

38 first_name = first_name.split(' ')[0]

39 full_name = first_name + ' ' + last_name

40 if len(full_name) <= max_length:

41 return full_name

42

43 # drop latter parts of combined first names

44 first_name = re.split(r'[\s\-]', first_name)[0]

45 full_name = first_name + ' ' + last_name

46 if len(full_name) <= max_length:

47 return full_name

48

49 # drop latter parts of multi part last names

50 last_name = re.split(r'[\s\-]', last_name)[0]

51 full_name = first_name + ' ' + last_name

52 if len(full_name) <= max_length:

53 return full_name

54

55 # shorten last name to one letter

56 last_name = last_name[:1]

57

58 full_name = first_name + ' ' + last_name

59 if len(full_name) > max_length:

60 raise Exception('Failed to shorten name {}'.format(original_full_name))

61 return full_name

62

63

64 def format_timedelta(dt: timedelta, days_label: str = 'd', hours_label: str = 'h',

65 minutes_label: str = 'min', seconds_label: str = 's') -> str:

66 """

67 Formats timedelta to readable format, e.g. 1h30min15s.

68 :param dt: timedelta

69 :param days_label: Label for days. Leave empty '' if value should be skipped / ignored.

70 :param hours_label: Label for hours. Leave empty '' if value should be skipped / ignored.

71 :param minutes_label: Label for minutes. Leave empty '' if value should be skipped / ignored.

72 :param seconds_label: Label for seconds. Leave empty '' if value should be skipped / ignored.

73 :return: str

74 """

75 parts = (

76 (86400, days_label),

77 (3600, hours_label),

78 (60, minutes_label),

79 (1, seconds_label),

80 )

81 out = ""

82 seconds = int(dt.total_seconds())

83 for n_secs, label in parts:

84 n, remainder = divmod(seconds, n_secs)

85 if n > 0 and label:

86 out += str(n) + label

87 seconds = remainder

88 return out.strip()

89

90

91 def format_xml(content: str, encoding: str = 'UTF-8', exceptions: bool = False) -> str:

92 """

93 Formats XML document as human-readable plain text.

94 If settings.XMLLINT_PATH is defined xmllint is used for formatting (higher quality). Otherwise minidom toprettyxml is used.

95 :param content: XML data as str

96 :param encoding: XML file encoding

97 :param exceptions: Raise exceptions on error

98 :return: str (Formatted XML str)

99 """

100 assert isinstance(content, str)

101 try:

102 if hasattr(settings, 'XMLLINT_PATH') and settings.XMLLINT_PATH: 102 ↛ 108line 102 didn't jump to line 108, because the condition on line 102 was never false

103 with tempfile.NamedTemporaryFile() as fp:

104 fp.write(content.encode(encoding=encoding))

105 fp.flush()

106 out = subprocess.check_output([settings.XMLLINT_PATH, '--format', fp.name])

107 return out.decode(encoding=encoding)

108 return xml.dom.minidom.parseString(content).toprettyxml()

109 except Exception as e:

110 logger.error('format_xml failed: %s', e)

111 if exceptions:

112 raise

113 return content

114

115

116 def format_xml_bytes(content: bytes, encoding: str = 'UTF-8', exceptions: bool = False) -> bytes:

117 """

118 Formats XML document as human-readable plain text and returns result in bytes.

119 If settings.XMLLINT_PATH is defined xmllint is used for formatting (higher quality). Otherwise minidom toprettyxml is used.

120 :param content: XML data as bytes

121 :param encoding: XML file encoding

122 :param exceptions: Raise exceptions on error

123 :return: bytes (Formatted XML as bytes)

124 """

125 assert isinstance(content, bytes)

126 try:

127 if hasattr(settings, 'XMLLINT_PATH') and settings.XMLLINT_PATH: 127 ↛ 133line 127 didn't jump to line 133, because the condition on line 127 was never false

128 with tempfile.NamedTemporaryFile() as fp:

129 fp.write(content)

130 fp.flush()

131 out = subprocess.check_output([settings.XMLLINT_PATH, '--format', fp.name])

132 return out

133 return xml.dom.minidom.parseString(content.decode(encoding=encoding)).toprettyxml(encoding=encoding)

134 except Exception as e:

135 logger.error('format_xml_bytes failed: %s', e)

136 if exceptions:

137 raise

138 return content

139

140

141 def format_xml_file(full_path: str, encoding: str = 'UTF-8', exceptions: bool = False) -> bytes:

142 """

143 Formats XML file as human-readable plain text and returns result in bytes.

144 Tries to format XML file first, if formatting fails the file content is returned as is.

145 If the file does not exist empty bytes is returned.

146 If settings.XMLLINT_PATH is defined xmllint is used for formatting (higher quality). Otherwise minidom toprettyxml is used.

147 :param full_path: Full path to XML file

148 :param encoding: XML file encoding

149 :param exceptions: Raise exceptions on error

150 :return: bytes

151 """

152 try:

153 if hasattr(settings, 'XMLLINT_PATH') and settings.XMLLINT_PATH:

154 return subprocess.check_output([settings.XMLLINT_PATH, '--format', full_path])

155 with open(full_path, 'rb') as fp:

156 return xml.dom.minidom.parse(fp).toprettyxml(encoding=encoding)

157 except Exception as e:

158 logger.error('format_xml_file failed (1): %s', e)

159 if exceptions:

160 raise

161 try:

162 with open(full_path, 'rb') as fp:

163 return fp.read()

164 except Exception as e:

165 logger.error('format_xml_file failed (2): %s', e)

166 return b''

167

168

169 def format_csv(rows: List[List[Any]], dialect: str = 'excel') -> str:

170 """

171 Formats rows to CSV string content.

172 :param rows: List[List[Any]]

173 :param dialect: See csv.writer dialect

174 :return: str

175 """

176 f = StringIO()

177 writer = csv.writer(f, dialect=dialect)

178 for row in rows:

179 writer.writerow(row)

180 return f.getvalue()

181

182

183 def format_table(rows: List[List[Any]], max_col: Optional[int] = None, max_line: Optional[int] = 200, # noqa

184 col_sep: str = '|', row_sep: str = '-', row_begin: str = '|', row_end: str = '|',

185 has_label_row: bool = False,

186 left_align: Optional[List[int]] = None, center_align: Optional[List[int]] = None) -> str:

187 """

188 Formats "ASCII-table" rows by padding column widths to longest column value, optionally limiting column widths.

189 Optionally separates colums with ' | ' character and header row with '-' characters.

190 Supports left, right and center alignment. Useful for console apps / debugging.

191

192 :param rows: List[List[Any]]

193 :param max_col: Max column value width. Pass None for unlimited length.

194 :param max_line: Maximum single line length. Exceeding columns truncated. Pass None for unlimited length.

195 :param col_sep: Column separator string.

196 :param row_sep: Row separator character used before first row, end, after first row (if has_label_row).

197 :param row_begin: Row begin string, inserted before each row.

198 :param row_end: Row end string, appended after each row.

199 :param has_label_row: Set to True if table starts with column label row.

200 :param left_align: Indexes of left-aligned columns. By default all are right aligned.

201 :param center_align: Indexes of center-aligned columns. By default all are right aligned.

202 :return: str

203 """

204 # validate parameters

205 assert max_col is None or max_col > 2

206 if left_align is None:

207 left_align = []

208 if center_align is None:

209 center_align = []

210 if left_align:

211 if set(left_align) & set(center_align): 211 ↛ 212line 211 didn't jump to line 212, because the condition on line 211 was never true

212 raise ValidationError('Left align columns {} overlap with center align {}'.format(left_align, center_align))

213

214 # find out number of columns

215 ncols = 0

216 for row in rows:

217 ncols = max(ncols, len(row))

218

219 # find out full-width column lengths

220 col_lens0: List[int] = [0] * ncols

221 for row in rows:

222 for ix, v in enumerate(row):

223 v = str(v)

224 col_lens0[ix] = max(col_lens0[ix], len(v))

225

226 # adjust max_col if needed

227 if max_line and (not max_col or sum(col_lens0) > max_line): 227 ↛ 228line 227 didn't jump to line 228, because the condition on line 227 was never true

228 max_col = max_line // ncols

229

230 # length limited lines and final column lengths

231 col_lens = [0] * ncols

232 lines: List[List[str]] = []

233 for row in rows:

234 line = []

235 for ix, v in enumerate(row):

236 v = str(v)

237 if max_col and len(v) > max_col:

238 v = v[:max_col-2] + '..'

239 line.append(v)

240 col_lens[ix] = max(col_lens[ix], len(v))

241 while len(line) < ncols: 241 ↛ 242line 241 didn't jump to line 242, because the condition on line 241 was never true

242 line.append('')

243 lines.append(line)

244

245 # padded lines

246 lines2: List[List[str]] = []

247 for line in lines:

248 line2 = []

249 for ix, v in enumerate(line):

250 col_len = col_lens[ix]

251 if len(v) < col_len:

252 if ix in left_align:

253 v = v + ' ' * (col_len - len(v))

254 elif ix in center_align:

255 pad = col_len - len(v)

256 lpad = int(pad/2)

257 rpad = pad - lpad

258 v = ' ' * lpad + v + ' '*rpad

259 else:

260 v = ' '*(col_len-len(v)) + v

261 line2.append(v)

262 lines2.append(line2)

263

264 # calculate max number of columns and max line length

265 max_line_len = 0

266 col_sep_len = len(col_sep)

267 ncols0 = ncols

268 for line in lines2:

269 if max_line is not None: 269 ↛ 268line 269 didn't jump to line 268, because the condition on line 269 was never false

270 line_len = len(row_begin) + sum(len(v)+col_sep_len for v in line[:ncols]) - col_sep_len + len(row_end)

271 while line_len > max_line:

272 ncols -= 1

273 line_len = len(row_begin) + sum(len(v)+col_sep_len for v in line[:ncols]) - col_sep_len + len(row_end)

274 max_line_len = max(max_line_len, line_len)

275

276 # find out how we should terminate lines/rows

277 line_term = ''

278 row_sep_term = ''

279 if ncols0 > ncols:

280 line_term = '..'

281 row_sep_term = row_sep * int(2 / len(row_sep))

282

283 # final output with row and column separators

284 lines3 = []

285 if row_sep: 285 ↛ 287line 285 didn't jump to line 287, because the condition on line 285 was never false

286 lines3.append(row_sep * max_line_len + row_sep_term)

287 for line_ix, line in enumerate(lines2):

288 while len(line) > ncols:

289 line.pop()

290 line_out = col_sep.join(line)

291 lines3.append(row_begin + line_out + row_end + line_term)

292 if line_ix == 0 and row_sep and has_label_row:

293 lines3.append(row_sep * max_line_len + row_sep_term)

294 if row_sep: 294 ↛ 296line 294 didn't jump to line 296, because the condition on line 294 was never false

295 lines3.append(row_sep * max_line_len + row_sep_term)

296 return '\n'.join(lines3)

297

298

299 def ucfirst(v: str) -> str:

300 """

301 Converts first character of the string to uppercase.

302 :param v: str

303 :return: str

304 """

305 return v[0:1].upper() + v[1:]

306

307

308 ucfirst_lazy = lazy(ucfirst, str)

309

310

311 def dec1(a: Union[float, int, Decimal, str]) -> Decimal:

312 """

313 Converts number to Decimal with 1 decimal digits.

314 :param a: Number

315 :return: Decimal with 1 decimal digits

316 """

317 return Decimal(a).quantize(Decimal('1.0'))

318

319

320 def dec2(a: Union[float, int, Decimal, str]) -> Decimal:

321 """

322 Converts number to Decimal with 2 decimal digits.

323 :param a: Number

324 :return: Decimal with 2 decimal digits

325 """

326 return Decimal(a).quantize(Decimal('1.00'))

327

328

329 def dec3(a: Union[float, int, Decimal, str]) -> Decimal:

330 """

331 Converts number to Decimal with 3 decimal digits.

332 :param a: Number

333 :return: Decimal with 3 decimal digits

334 """

335 return Decimal(a).quantize(Decimal('1.000'))

336

337

338 def dec4(a: Union[float, int, Decimal, str]) -> Decimal:

339 """

340 Converts number to Decimal with 4 decimal digits.

341 :param a: Number

342 :return: Decimal with 4 decimal digits

343 """

344 return Decimal(a).quantize(Decimal('1.0000'))

345

346

347 def dec5(a: Union[float, int, Decimal, str]) -> Decimal:

348 """

349 Converts number to Decimal with 5 decimal digits.

350 :param a: Number

351 :return: Decimal with 4 decimal digits

352 """

353 return Decimal(a).quantize(Decimal('1.00000'))

354

355

356 def dec6(a: Union[float, int, Decimal, str]) -> Decimal:

357 """

358 Converts number to Decimal with 6 decimal digits.

359 :param a: Number

360 :return: Decimal with 4 decimal digits

361 """

362 return Decimal(a).quantize(Decimal('1.000000'))

363

364

365 def strip_media_root(file_path: str) -> str:

366 """

367 If file path starts with (settings) MEDIA_ROOT,

368 the MEDIA_ROOT part gets stripped and only relative path is returned.

369 Otherwise file path is returned as is. This enabled stored file names in more

370 portable format for different environment / storage.

371 If MEDIA_ROOT is missing or empty, the filename is returned as is.

372 Reverse operation of this is get_media_full_path().

373 :param file_path: str

374 :return: str

375 """

376 if hasattr(settings, 'MEDIA_ROOT') and settings.MEDIA_ROOT and file_path.startswith(settings.MEDIA_ROOT):

377 file_path = file_path[len(settings.MEDIA_ROOT):]

378 if file_path.startswith('/'): 378 ↛ 380line 378 didn't jump to line 380, because the condition on line 378 was never false

379 return file_path[1:]

380 return file_path

381

382

383 def get_media_full_path(file_path: str) -> str:

384 """

385 If file path is absolute, the path is returned as is.

386 Otherwise it is returned relative to (settings) MEDIA_ROOT.

387 This enabled stored file names in more portable format for different environment / storage.

388 If MEDIA_ROOT is missing or empty, the filename is returned as is.

389 Reverse operation of this is strip_media_root().

390 :param file_path: str

391 :return: str

392 """

393 if file_path and not os.path.isabs(file_path):

394 return os.path.join(settings.MEDIA_ROOT, file_path)

395 return file_path

396

397

398 def camel_case_to_underscore(s: str) -> str:

399 """

400 Converts camelCaseWord to camel_case_word.

401 :param s: str

402 :return: str

403 """

404 if s: 404 ↛ 408line 404 didn't jump to line 408, because the condition on line 404 was never false

405 s = re.sub(r"([A-Z]+)([A-Z][a-z])", r'\1_\2', s)

406 s = re.sub(r"([a-z\d])([A-Z])", r'\1_\2', s)

407 s = s.replace("-", "_")

408 return s.lower()

409

410

411 def underscore_to_camel_case(s: str) -> str:

412 """

413 Converts under_score_word to underScoreWord.

414 :param s: str

415 :return: str

416 """

417 if s: 417 ↛ 420line 417 didn't jump to line 420, because the condition on line 417 was never false

418 p = s.split('_')

419 s = p[0] + ''.join([ucfirst(w) for w in p[1:]])

420 return s