Coverage for jbank/parsers.py: 87%

# Coverage for jbank/parsers.py : 87%

## 307 statements

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1 import logging

2 import re

3 from copy import copy

4 from os.path import basename

5 from datetime import time, datetime, date

6 from decimal import Decimal

7 from typing import List, Any, Tuple, Optional, Dict, Sequence, Union

8 from django.core.exceptions import ValidationError

9 from django.utils.translation import gettext as _

10 from pytz import timezone

11

12

13 REGEX_SIMPLE_FIELD = re.compile(r'^(X|9)+$')

14

15 REGEX_VARIABLE_FIELD = re.compile(r'^(X|9)\((\d+)\)$')

16

17 TO_STATEMENT_SUFFIXES = ('TO', 'TXT')

18

19 TO_FILE_HEADER_TYPES = ('T00', )

20

21 TO_FILE_HEADER_DATES = (

22 'begin_date',

23 'end_date',

24 ('record_date', 'record_time'),

25 'begin_balance_date',

26)

27

28 TO_FILE_HEADER_DECIMALS = (

29 ('begin_balance', 'begin_balance_sign'),

30 'account_limit',

31)

32

33 TO_FILE_HEADER = (

34 ('statement_type', 'X', 'P'),

35 ('record_type', 'XX', 'P'),

36 ('record_length', '9(3)', 'P'),

37 ('version', 'X(3)', 'P'),

38 ('account_number', 'X(14)', 'P'),

39 ('statement_number', '9(3)', 'P'),

40 ('begin_date', '9(6)', 'P'),

41 ('end_date', '9(6)', 'P'),

42 ('record_date', '9(6)', 'P'),

43 ('record_time', '9(4)', 'P'),

44 ('customer_identifier', 'X(17)', 'P'),

45 ('begin_balance_date', '9(6)', 'P'),

46 ('begin_balance_sign', 'X', 'P'),

47 ('begin_balance', '9(18)', 'P'),

48 ('record_count', '9(6)', 'P'),

49 ('currency_code', 'X(3)', 'P'),

50 ('account_name', 'X(30)', 'V'),

51 ('account_limit', '9(18)', 'P'),

52 ('owner_name', 'X(35)', 'P'),

53 ('contact_info_1', 'X(40)', 'P'),

54 ('contact_info_2', 'X(40)', 'V'),

55 ('bank_specific_info_1', 'X(30)', 'V'),

56 ('iban_and_bic', 'X(30)', 'V'),

57)

58

59 TO_FILE_RECORD_TYPES = ('T10', 'T80')

60

61 TO_FILE_RECORD_DATES = (

62 'record_date',

63 'value_date',

64 'paid_date',

65)

66

67 TO_FILE_RECORD_DECIMALS = (

68 ('amount', 'amount_sign'),

69)

70

71 TO_FILE_RECORD = (

72 ('statement_type', 'X', 'P'),

73 ('record_type', 'XX', 'P'),

74 ('record_length', '9(3)', 'P'),

75 ('record_number', '9(6)', 'P'),

76 ('archive_identifier', 'X(18)', 'V'),

77 ('record_date', '9(6)', 'P'),

78 ('value_date', '9(6)', 'V'),

79 ('paid_date', '9(6)', 'V'),

80 ('entry_type', 'X', 'P'), # 1 = pano, 2 = otto, 3 = panon korjaus, 4 = oton korjaus, 9 = hylätty tapahtuma

81 ('record_code', 'X(3)', 'P'),

82 ('record_description', 'X(35)', 'P'),

83 ('amount_sign', 'X', 'P'),

84 ('amount', '9(18)', 'P'),

85 ('receipt_code', 'X', 'P'),

86 ('delivery_method', 'X', 'P'),

87 ('name', 'X(35)', 'V'),

88 ('name_source', 'X', 'V'),

89 ('recipient_account_number', 'X(14)', 'V'),

90 ('recipient_account_number_changed', 'X', 'V'),

91 ('remittance_info', 'X(20)', 'V'),

92 ('form_number', 'X(8)', 'V'),

93 ('level_identifier', 'X', 'P'),

94)

95

96 TO_FILE_RECORD_EXTRA_INFO_TYPES = ('T11', 'T81')

97

98 TO_FILE_RECORD_EXTRA_INFO_HEADER = (

99 ('statement_type', 'X', 'P'),

100 ('record_type', 'XX', 'P'),

101 ('record_length', '9(3)', 'P'),

102 ('extra_info_type', '9(2)', 'P'),

103)

104

105 TO_FILE_RECORD_EXTRA_INFO_COUNTS = (

106 ('entry_count', '9(8)', 'P'),

107)

108

109 TO_FILE_RECORD_EXTRA_INFO_INVOICE = (

110 ('customer_number', 'X(10)', 'P'),

111 ('pad01', 'X', 'P'),

112 ('invoice_number', 'X(15)', 'P'),

113 ('pad02', 'X', 'P'),

114 ('invoice_date', 'X(6)', 'P'),

115)

116

117 TO_FILE_RECORD_EXTRA_INFO_CARD = (

118 ('card_number', 'X(19)', 'P'),

119 ('pad01', 'X', 'P'),

120 ('merchant_reference', 'X(14)', 'P'),

121)

122

123 TO_FILE_RECORD_EXTRA_INFO_CORRECTION = (

124 ('original_archive_identifier', 'X(18)', 'P'),

125)

126

127 TO_FILE_RECORD_EXTRA_INFO_CURRENCY_DECIMALS = (

128 ('amount', 'amount_sign'),

129)

130

131 TO_FILE_RECORD_EXTRA_INFO_CURRENCY = (

132 ('amount_sign', 'X', 'P'),

133 ('amount', '9(18)', 'P'),

134 ('pad01', 'X', 'P'),

135 ('currency_code', 'X(3)', 'P'),

136 ('pad02', 'X', 'P'),

137 ('exchange_rate', '9(11)', 'P'),

138 ('rate_reference', 'X(6)', 'V'),

139)

140

141 TO_FILE_RECORD_EXTRA_INFO_REASON = (

142 ('reason_code', '9(3)', 'P'),

143 ('pad01', 'X', 'P'),

144 ('reason_description', 'X(31)', 'P'),

145)

146

147 TO_FILE_RECORD_EXTRA_INFO_NAME_DETAIL = (

148 ('name_detail', 'X(35)', 'P'),

149)

150

151 TO_FILE_RECORD_EXTRA_INFO_SEPA = (

152 ('reference', 'X(35)', 'V'),

153 ('iban_account_number', 'X(35)', 'V'),

154 ('bic_code', 'X(35)', 'V'),

155 ('recipient_name_detail', 'X(70)', 'V'),

156 ('payer_name_detail', 'X(70)', 'V'),

157 ('identifier', 'X(35)', 'V'),

158 ('archive_identifier', 'X(35)', 'V'),

159)

160

161 TO_BALANCE_RECORD_DATES = (

162 'record_date',

163)

164

165 TO_BALANCE_RECORD_DECIMALS = (

166 ('end_balance', 'end_balance_sign'),

167 ('available_balance', 'available_balance_sign'),

168)

169

170 TO_BALANCE_RECORD = (

171 ('statement_type', 'X', 'P'),

172 ('record_type', 'XX', 'P'),

173 ('record_length', '9(3)', 'P'),

174 ('record_date', '9(6)', 'P'),

175 ('end_balance_sign', 'X', 'P'),

176 ('end_balance', '9(18)', 'P'),

177 ('available_balance_sign', 'X', 'P'),

178 ('available_balance', '9(18)', 'P'),

179)

180

181 TO_CUMULATIVE_RECORD_DATES = (

182 'period_date',

183)

184

185 TO_CUMULATIVE_RECORD_DECIMALS = (

186 ('deposits_amount', 'deposits_sign'),

187 ('withdrawals_amount', 'withdrawals_sign'),

188)

189

190 TO_CUMULATIVE_RECORD = (

191 ('statement_type', 'X', 'P'),

192 ('record_type', 'XX', 'P'),

193 ('record_length', '9(3)', 'P'),

194 ('period_identifier', 'X', 'P'), # 1=day, 2=term, 3=month, 4=year

195 ('period_date', '9(6)', 'P'),

196 ('deposits_count', '9(8)', 'P'),

197 ('deposits_sign', 'X', 'P'),

198 ('deposits_amount', '9(18)', 'P'),

199 ('withdrawals_count', '9(8)', 'P'),

200 ('withdrawals_sign', 'X', 'P'),

201 ('withdrawals_amount', '9(18)', 'P'),

202)

203

204 TO_SPECIAL_RECORD = (

205 ('statement_type', 'X', 'P'),

206 ('record_type', 'XX', 'P'),

207 ('record_length', '9(3)', 'P'),

208 ('bank_group_identifier', 'X(3)', 'P'),

209)

210

211 SVM_STATEMENT_SUFFIXES = ('SVM', 'TXT', 'KTL')

212

213 SVM_FILE_HEADER_DATES = (

214 ('record_date', 'record_time'),

215)

216

217 SVM_FILE_HEADER_TYPES = (

218 '0',

219)

220

221 SVM_FILE_HEADER = (

222 ('statement_type', '9(1)', 'P'),

223 ('record_date', '9(6)', 'P'),

224 ('record_time', '9(4)', 'P'),

225 ('institution_identifier', 'X(2)', 'P'),

226 ('service_identifier', 'X(9)', 'P'),

227 ('currency_identifier', 'X(1)', 'P'),

228 ('pad01', 'X(67)', 'P'),

229)

230

231 SVM_FILE_RECORD_TYPES = ('3', '5')

232

233 SVM_FILE_RECORD_DECIMALS = (

234 'amount',

235)

236

237 SVM_FILE_RECORD_DATES = (

238 'record_date',

239 'paid_date',

240)

241

242 SVM_FILE_RECORD = (

243 ('record_type', '9(1)', 'P'), # 3=viitesiirto, 5=suoraveloitus

244 ('account_number', '9(14)', 'P'),

245 ('record_date', '9(6)', 'P'),

246 ('paid_date', '9(6)', 'P'),

247 ('archive_identifier', 'X(16)', 'P'),

248 ('remittance_info', '9(20)', 'P'),

249 ('payer_name', 'X(12)', 'P'),

250 ('currency_identifier', 'X(1)', 'P'), # 1=eur

251 ('name_source', 'X', 'V'),

252 ('amount', '9(10)', 'P'),

253 ('correction_identifier', 'X', 'V'), # 0=normal, 1=correction

254 ('delivery_method', 'X', 'P'), # A=asiakkaalta, K=konttorista, J=pankin jarjestelmasta

255 ('receipt_code', 'X', 'P'),

256)

257

258 SVM_FILE_SUMMARY_TYPES = (

259 '9',

260)

261

262 SVM_FILE_SUMMARY_DECIMALS = (

263 'record_amount',

264 'correction_amount',

265)

266

267 SVM_FILE_SUMMARY = (

268 ('record_type', '9(1)', 'P'), # 9

269 ('record_count', '9(6)', 'P'),

270 ('record_amount', '9(11)', 'P'),

271 ('correction_count', '9(6)', 'P'),

272 ('correction_amount', '9(11)', 'P'),

273 ('pad01', 'X(5)', 'P'),

274)

275

276

277 logger = logging.getLogger(__name__)

278

279

280 def parse_record_format(fmt: str) -> tuple:

281 """

282 :param fmt: Data format used in .TO files

283 :return: Data type ('X' or '9'), data length (number of characters)

284 """

285 res = REGEX_SIMPLE_FIELD.match(fmt)

286 data_type, data_len = None, None

287 if res:

288 data_type = res.group(1)

289 data_len = len(fmt)

290 else:

291 res = REGEX_VARIABLE_FIELD.match(fmt)

292 if res:

293 data_type = res.group(1)

294 data_len = int(res.group(2))

295 if not data_type or not data_len:

296 raise Exception('Failed to parse data format {}'.format(fmt))

297 return data_type, data_len

298

299

300 def parse_record_value(data_type, data_len, data, name: str, line_number: int) -> str:

301 value = data[:data_len]

302 if len(value) != data_len:

303 raise ValidationError(_('Line {line}: Invalid field "{field}" value "{value}"').format(line=line_number, field=name, value=value))

304 if data_type == 'X':

305 pass

306 elif data_type == '9':

307 charset = '0123456789'

308 for ch in value:

309 if ch not in charset:

310 raise ValidationError(_('Line {line}: Invalid field "{field}" value "{value}"').format(line=line_number, field=name, value=value))

311 # logger.info('jbank.parsers.parse_record_value: {} = {}'.format(name, value))

312 else:

313 raise ValidationError(_('Line {line}: Invalid field "{field}" value "{value}"').format(line=line_number, field=name, value=value))

314 return value

315

316

317 def parse_records(line: str, specs: Sequence[Tuple[str, str, str]], line_number: int, check_record_length: bool = True,

318 record_length: Optional[int] = None) -> Dict[str, Union[int, str]]:

319 i = 0

320 data: Dict[str, Union[int, str]] = dict()

321 data['line_number'] = line_number

322 for name, fmt, req in specs: # pylint: disable=unused-variable

323 data_type, data_len = parse_record_format(fmt)

324 value = parse_record_value(data_type, data_len, line[i:], name=name, line_number=line_number)

325 # print('[{}:{}] {}="{}"'.format(i, i+data_len, name, value))

326 data[name] = str(value).strip()

327 i += data_len

328 data['extra_data'] = line[i:]

329

330 rec_len = data.get('record_length', record_length)

331 if check_record_length and rec_len:

332 data['extra_data'] = str(data['extra_data']).strip()

333 if i != rec_len and data['extra_data'] != '':

334 raise ValidationError(_('Line {line}: Record length ({record_length}) does not match length of '

335 'parsed data ({data_length}). Extra data: "{extra_data}"').format(

336 line=line_number, data_length=i+len(str(data['extra_data'])),

337 record_length=rec_len, extra_data=data['extra_data']))

338 return data

339

340

341 def parse_record_messages(extra_data: str) -> List[str]:

342 msg = []

343 while extra_data:

344 msg.append(extra_data[:35])

345 extra_data = extra_data[35:]

346 return msg

347

348

349 def parse_record_extra_info(record: Dict[str, Any], line: str, line_number: int):

350 if line[:3] not in TO_FILE_RECORD_EXTRA_INFO_TYPES:

351 raise ValidationError('SVM record extra info validation error on line {}'.format(line_number))

352

353 header = parse_records(line, TO_FILE_RECORD_EXTRA_INFO_HEADER, line_number, check_record_length=False)

354 extra_info_type = header['extra_info_type']

355 # print(line)

356 extra_data = copy(header['extra_data'])

357 assert isinstance(extra_data, str)

358 if extra_info_type == '00':

359 record['messages'] = parse_record_messages(extra_data)

360 elif extra_info_type == '01':

361 record['counts'] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_COUNTS, line_number, record_length=8)

362 elif extra_info_type == '02':

363 record['invoice'] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_INVOICE, line_number, record_length=33)

364 elif extra_info_type == '03':

365 record['card'] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_CARD, line_number, record_length=34)

366 elif extra_info_type == '04':

367 record['correction'] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_CORRECTION, line_number, record_length=18)

368 elif extra_info_type == '05':

369 record['currency'] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_CURRENCY, line_number, record_length=41)

370 convert_decimal_fields(record['currency'], TO_FILE_RECORD_EXTRA_INFO_CURRENCY_DECIMALS)

371 elif extra_info_type == '06':

372 record['client_messages'] = parse_record_messages(extra_data)

373 elif extra_info_type == '07':

374 record['bank_messages'] = parse_record_messages(extra_data)

375 elif extra_info_type == '08':

376 record['reason'] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_REASON, line_number, record_length=35)

377 elif extra_info_type == '09':

378 record['name_detail'] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_NAME_DETAIL, line_number, record_length=35)

379 elif extra_info_type == '11':

380 record['sepa'] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_SEPA, line_number, record_length=323)

381 else:

382 raise ValidationError(_('Line {line}: Invalid record extra info type "{extra_info_type}"').format(line=line_number, extra_info_type=extra_info_type))

383

384

385 def convert_date(v: Optional[str], field_name: str) -> date:

386 if v is None:

387 raise ValidationError(_("Date field missing: {}").format(field_name))

388 if len(v) != 6:

389 raise ValidationError(_("Date format error in field {}: {}").format(field_name, v))

390 year = int(v[0:2]) + 2000

391 month = int(v[2:4])

392 day = int(v[4:6])

393 return date(year=year, month=month, day=day)

394

395

396 def convert_time(v: Optional[str], field_name: str) -> time:

397 if v is None:

398 raise ValidationError(_("Time field missing: {}").format(field_name))

399 if not re.match(r'^\d\d\d\d$', v):

400 raise ValidationError(_("Time format error in field {}: {}").format(field_name, v))

401 return time(int(v[0:2]), int(v[2:4]))

402

403

404 def convert_date_fields(data: dict, date_fields: tuple, tz: Any):

405 for k in date_fields:

406 if isinstance(k, str):

407 v_date = data.get(k)

408 if v_date:

409 data[k] = convert_date(v_date, k)

410 elif isinstance(k, tuple):

411 if len(k) != 2:

412 raise ValidationError(_("Date format error in field {}").format(k))

413 k_date, k_time = k

414 v_date, v_time = data.get(k_date), data.get(k_time)

415 # print('Converting {}'.format(k))

416 # pprint(data)

417 if v_date or v_time:

418 assert v_date is None or isinstance(v_date, str)

419 assert v_time is None or isinstance(v_time, str)

420 v_date = convert_date(v_date, k_date)

421 v_time = convert_time(v_time, k_time)

422 v_datetime = datetime.combine(v_date, v_time)

423 data[k_date] = tz.localize(v_datetime)

424 del data[k_time]

425

426

427 def convert_decimal_fields(data: dict, decimal_fields: tuple):

428 for field in decimal_fields:

429 if isinstance(field, str):

430 v_number = data.get(field)

431 if v_number is not None:

432 v = Decimal(v_number) * Decimal('0.01')

433 # logger.info('jbank.parsers.convert_decimal_fields: {} = {}'.format(field, v))

434 data[field] = v

435 elif isinstance(field, tuple) and len(field) == 2:

436 k_number, k_sign = field

437 v_number, v_sign = data.get(k_number), data.get(k_sign)

438 if v_number is not None:

439 v = Decimal(v_number) * Decimal('0.01')

440 if v_sign == '-':

441 v = -v

442 data[k_number] = v

443 # logger.info('jbank.parsers.convert_decimal_fields: {} = {}'.format(k_number, v))

444 del data[k_sign]

445 else:

446 raise ValidationError(_('Invalid decimal field format: {}').format(field))

447

448

449 def combine_statement(header, records, balance, cumulative, cumulative_adjustment, special_records) -> dict: # pylint: disable=too-many-arguments

450 data = {

451 'header': header,

452 'records': records,

453 }

454 if balance is not None:

455 data['balance'] = balance

456 if cumulative is not None:

457 data['cumulative'] = cumulative

458 if cumulative_adjustment is not None:

459 data['cumulative_adjustment'] = cumulative_adjustment

460 if special_records:

461 data['special_records'] = special_records

462 return data

463

464

465 def parse_tiliote_statements(content: str, filename: str) -> list: # pylint: disable=too-many-locals

466 lines = content.split('\n')

467 nlines = len(lines)

468

469 line_number = 1

470 tz = timezone('Europe/Helsinki')

471

472 header = None

473 records = []

474 balance = None

475 cumulative = None

476 cumulative_adjustment = None

477 special_records = []

478 statements = []

479

480 while line_number <= nlines:

481 line = lines[line_number-1]

482 if line.strip() == '':

483 line_number += 1

484 continue

485 # print('parsing line {}: {}'.format(line_number, line))

486 record_type = line[:3]

487

488 if record_type in TO_FILE_HEADER_TYPES:

489 if header:

490 statements.append(combine_statement(header, records, balance, cumulative, cumulative_adjustment, special_records))

491 header, records, balance, cumulative, cumulative_adjustment, special_records = None, [], None, None, None, []

492

493 header = parse_records(lines[line_number - 1], TO_FILE_HEADER, line_number=line_number)

494 convert_date_fields(header, TO_FILE_HEADER_DATES, tz)

495 convert_decimal_fields(header, TO_FILE_HEADER_DECIMALS)

496 iban_and_bic = str(header.get('iban_and_bic', '')).split(' ')

497 if len(iban_and_bic) == 2:

498 header['iban'], header['bic'] = iban_and_bic

499 line_number += 1

500

501 elif record_type in TO_FILE_RECORD_TYPES:

502 record = parse_records(line, TO_FILE_RECORD, line_number=line_number)

503 convert_date_fields(record, TO_FILE_RECORD_DATES, tz)

504 convert_decimal_fields(record, TO_FILE_RECORD_DECIMALS)

505

506 line_number += 1

507 # check for record extra info

508 if line_number <= nlines:

509 line = lines[line_number-1]

510 while line[:3] in TO_FILE_RECORD_EXTRA_INFO_TYPES:

511 parse_record_extra_info(record, line, line_number=line_number)

512 line_number += 1

513 line = lines[line_number-1]

514

515 records.append(record)

516 elif record_type == 'T40':

517 balance = parse_records(line, TO_BALANCE_RECORD, line_number=line_number)

518 convert_date_fields(balance, TO_BALANCE_RECORD_DATES, tz)

519 convert_decimal_fields(balance, TO_BALANCE_RECORD_DECIMALS)

520 line_number += 1

521 elif record_type == 'T50':

522 cumulative = parse_records(line, TO_CUMULATIVE_RECORD, line_number=line_number)

523 convert_date_fields(cumulative, TO_CUMULATIVE_RECORD_DATES, tz)

524 convert_decimal_fields(cumulative, TO_CUMULATIVE_RECORD_DECIMALS)

525 line_number += 1

526 elif record_type == 'T51':

527 cumulative_adjustment = parse_records(line, TO_CUMULATIVE_RECORD, line_number=line_number)

528 convert_date_fields(cumulative_adjustment, TO_CUMULATIVE_RECORD_DATES, tz)

529 convert_decimal_fields(cumulative_adjustment, TO_CUMULATIVE_RECORD_DECIMALS)

530 line_number += 1

531 elif record_type in ('T60', 'T70'):

532 special_records.append(parse_records(line, TO_SPECIAL_RECORD, line_number=line_number, check_record_length=False))

533 line_number += 1

534 else:

535 raise ValidationError(_('Unknown record type on {}({}): {}').format(filename, line_number, record_type))

536

537 statements.append(combine_statement(header, records, balance, cumulative, cumulative_adjustment, special_records))

538 return statements

539

540

541 def parse_filename_suffix(filename: str) -> str:

542 a = filename.rsplit('.', 1)

543 return a[len(a)-1]

544

545

546 def parse_tiliote_statements_from_file(filename: str) -> list:

547 if parse_filename_suffix(filename).upper() not in TO_STATEMENT_SUFFIXES:

548 raise ValidationError(_('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format(

549 filename=filename, suffixes=', '.join(TO_STATEMENT_SUFFIXES), file_type='tiliote'))

550 with open(filename, 'rt', encoding='ISO-8859-1') as fp:

551 return parse_tiliote_statements(fp.read(), filename=basename(filename)) # type: ignore

552

553

554 def combine_svm_batch(header: Optional[Dict[str, Any]], records: List[Dict[str, Union[int, str]]],

555 summary: Optional[Dict[str, Any]]) -> Dict[str, Any]:

556 data = {'header': header, 'records': records}

557 if summary is not None:

558 data['summary'] = summary

559 return data

560

561

562 def parse_svm_batches(content: str, filename: str) -> list:

563 lines = content.split('\n')

564 nlines = len(lines)

565

566 line_number = 1

567 tz = timezone('Europe/Helsinki')

568

569 batches = []

570 header: Union[Dict[str, Union[int, str]], None] = None

571 summary: Union[Dict[str, Union[int, str]], None] = None

572 records: List[Dict[str, Union[int, str]]] = []

573

574 while line_number <= nlines:

575 line = lines[line_number-1]

576 if line.strip() == '':

577 line_number += 1

578 continue

579 record_type = line[:1]

580

581 if record_type in SVM_FILE_HEADER_TYPES:

582 if header:

583 batches.append(combine_svm_batch(header, records, summary))

584 header, records, summary = None, [], None

585 header = parse_records(lines[line_number - 1], SVM_FILE_HEADER, line_number=line_number)

586 convert_date_fields(header, SVM_FILE_HEADER_DATES, tz)

587 line_number += 1

588 elif record_type in SVM_FILE_RECORD_TYPES:

589 record = parse_records(line, SVM_FILE_RECORD, line_number=line_number)

590 convert_date_fields(record, SVM_FILE_RECORD_DATES, tz)

591 convert_decimal_fields(record, SVM_FILE_RECORD_DECIMALS)

592 line_number += 1

593 records.append(record)

594 elif record_type in SVM_FILE_SUMMARY_TYPES:

595 summary = parse_records(line, SVM_FILE_SUMMARY, line_number=line_number)

596 convert_decimal_fields(summary, SVM_FILE_SUMMARY_DECIMALS)

597 line_number += 1

598 else:

599 raise ValidationError(_('Unknown record type on {}({}): {}').format(filename, line_number, record_type))

600

601 batches.append(combine_svm_batch(header, records, summary))

602 return batches

603

604

605 def parse_svm_batches_from_file(filename: str) -> list:

606 if parse_filename_suffix(filename).upper() not in SVM_STATEMENT_SUFFIXES:

607 raise ValidationError(_('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format(

608 filename=filename, suffixes=', '.join(SVM_STATEMENT_SUFFIXES), file_type='saapuvat viitemaksut'))

609 with open(filename, 'rt', encoding='ISO-8859-1') as fp:

610 return parse_svm_batches(fp.read(), filename=basename(filename)) # type: ignore