Coverage for jbank/camt.py: 85%

1 import logging

2 from datetime import datetime, date

3 from decimal import Decimal

4 from typing import Tuple, Any, Optional, Dict

5

6 from django.conf import settings

7 from django.core.exceptions import ValidationError

8 from django.db import transaction

9 from django.utils.dateparse import parse_date

10 from django.utils.translation import gettext as _

11 from jacc.models import Account, EntryType

12 from jutil.format import dec2, dec4

13 from jutil.model import clone_model

14 from jutil.parse import parse_datetime

15 from jbank.models import (

16 StatementFile,

17 Statement,

18 StatementRecord,

19 DELIVERY_FROM_BANK_SYSTEM,

20 StatementRecordDetail,

21 CurrencyExchange,

22 StatementRecordRemittanceInfo,

23 CurrencyExchangeSource, ReferencePaymentBatchFile, ReferencePaymentBatch, ReferencePaymentRecord,

24)

25 from jbank.parsers import parse_filename_suffix

26 from jutil.xml import xml_to_dict

27

28

29 CAMT053_STATEMENT_SUFFIXES = ("XML", "XT", "CAMT", "NDCAMT53L", "XML")

30

31 CAMT053_ARRAY_TAGS = ["Bal", "Ntry", "NtryDtls", "TxDtls", "Strd", "Ustrd"]

32

33 CAMT053_INT_TAGS = ["NbOfNtries", "NbOfTxs"]

34

35 CAMT054_STATEMENT_SUFFIXES = ("XE", "XE", "CAMT", "NDCAMT54L", "XML")

36

37 CAMT054_ARRAY_TAGS = ["Ntfctn", "Othr", "Ntry", "NtryDtls", "PrtryAmt", "Chrgs", "AdrLine", "Strd", "Ustrd", "RfrdDocInf", "AddtlRmtInf"]

38

39 CAMT054_INT_TAGS = ["NbOfNtries", "NbOfTxs"]

40

41 logger = logging.getLogger(__name__)

42

43

44 def camt053_get_iban(data: dict) -> str:

45 return data.get("BkToCstmrStmt", {}).get("Stmt", {}).get("Acct", {}).get("Id", {}).get("IBAN", "")

46

47

48 def camt053_get_val(data: dict, key: str, default: Any = None, required: bool = True, name: str = "") -> Any:

49 if key not in data:

50 if required:

51 raise ValidationError(_("camt.053 field {} missing").format(name if name else key))

52 return default

53 return data[key]

54

55

56 def camt053_get_str(data: dict, key: str, default: str = "", required: bool = True, name: str = "") -> str:

57 return str(camt053_get_val(data, key, default, required, name))

58

59

60 def camt053_get_currency(data: dict, key: str, required: bool = True, name: str = "") -> Tuple[Optional[Decimal], str]:

61 try:

62 v = camt053_get_val(data, key, default=None, required=False, name=name)

63 if v is not None:

64 amount = dec2(v["@"])

65 currency_code = v["@Ccy"]

66 return amount, currency_code

67 except Exception:

68 pass

69 if required:

70 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "currency"))

71 return None, ""

72

73

74 def camt053_get_dt(data: Dict[str, Any], key: str, name: str = "") -> datetime:

75 s = camt053_get_val(data, key, None, True, name)

76 val = parse_datetime(s)

77 if val is None:

78 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "datetime") + ": {}".format(s))

79 return val

80

81

82 def camt053_get_int(data: Dict[str, Any], key: str, name: str = "") -> int:

83 s = camt053_get_val(data, key, None, True, name)

84 try:

85 return int(s)

86 except Exception:

87 pass

88 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "int"))

89

90

91 def camt053_get_int_or_none(data: Dict[str, Any], key: str, name: str = "") -> Optional[int]:

92 s = camt053_get_val(data, key, None, False, name)

93 if s is None:

94 return None

95 try:

96 return int(s)

97 except Exception:

98 pass

99 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "int"))

100

101

102 def camt053_get_date(data: dict, key: str, default: Optional[date] = None, required: bool = True, name: str = "") -> date:

103 s = camt053_get_val(data, key, default, required, name)

104 try:

105 val = parse_date(s[:10])

106 if val is None:

107 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "date"))

108 assert isinstance(val, date)

109 return val

110 except Exception:

111 pass

112 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "date") + ": {}".format(s))

113

114

115 def camt053_parse_statement_from_file(filename: str) -> dict:

116 if parse_filename_suffix(filename).upper() not in CAMT053_STATEMENT_SUFFIXES:

117 raise ValidationError(

118 _('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format(

119 filename=filename, suffixes=", ".join(CAMT053_STATEMENT_SUFFIXES), file_type="camt.053"

120 )

121 )

122 with open(filename, "rb") as fp:

123 data = xml_to_dict(fp.read(), array_tags=CAMT053_ARRAY_TAGS, int_tags=CAMT053_INT_TAGS)

124 return data

125

126

127 def camt053_get_stmt_bal(d_stmt: dict, bal_type: str) -> Tuple[Decimal, Optional[date]]:

128 for bal in d_stmt.get("Bal", []):

129 if bal.get("Tp", {}).get("CdOrPrtry", {}).get("Cd", "") == bal_type:

130 amt = Decimal(bal.get("Amt", {}).get("@", ""))

131 dt_data = bal.get("Dt", {})

132 dt = None

133 if "Dt" in dt_data:

134 dt = camt053_get_date(dt_data, "Dt", name="Stmt.Bal[{}].Dt.Dt".format(bal_type))

135 return amt, dt

136 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format("Stmt.Bal.Tp.CdOrPrty.Cd", bal_type))

137

138

139 def camt053_domain_from_record_code(record_domain: str) -> str:

140 if record_domain == "PMNT":

141 return "700"

142 if record_domain == "LDAS":

143 return "761"

144 return ""

145

146

147 def camt053_get_unified_val(qs, k: str, default: Any) -> Any:

148 v = default

149 for e in qs:

150 v2 = getattr(e, k)

151 if v == default:

152 v = v2

153 elif v and v2 and v2 != v:

154 return default

155 return v

156

157

158 def camt053_get_unified_str(qs, k: str) -> str:

159 return camt053_get_unified_val(qs, k, "")

160

161

162@transaction.atomic # noqa

163 def camt053_create_statement(statement_data: dict, name: str, file: StatementFile, **kw) -> Statement: # noqa

164 """

165 Creates camt.053 Statement from statement data parsed by camt053_parse_statement_from_file()

166 :param statement_data: XML data in form of dict

167 :param name: File name of the account statement

168 :param file: Source statement file

169 :return: Statement

170 """

171 account_number = camt053_get_iban(statement_data)

172 if not account_number:

173 raise ValidationError("{name}: ".format(name=name) + _("account.not.found").format(account_number=""))

174 accounts = list(Account.objects.filter(name=account_number))

175 if len(accounts) != 1:

176 raise ValidationError("{name}: ".format(name=name) + _("account.not.found").format(account_number=account_number) + " (" + str(len(accounts)) + ")")

177 account = accounts[0]

178 assert isinstance(account, Account)

179

180 d_stmt = statement_data.get("BkToCstmrStmt", {}).get("Stmt", {})

181 if not d_stmt:

182 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format("Stmt", "element"))

183 d_acct = d_stmt.get("Acct", {})

184 d_ownr = d_acct.get("Ownr", {})

185 d_ntry = d_stmt.get("Ntry", [])

186 d_frto = d_stmt.get("FrToDt", {})

187 d_txsummary = d_stmt.get("TxsSummry", {})

188

189 if Statement.objects.filter(name=name, account=account).first():

190 raise ValidationError("Bank account {} statement {} of processed already".format(account_number, name))

191 stm = Statement(name=name, account=account, file=file)

192 stm.account_number = stm.iban = account_number

193 stm.bic = camt053_get_str(d_acct.get("Svcr", {}).get("FinInstnId", {}), "BIC", name="Stmt.Acct.Svcr.FinInstnId.BIC")

194 stm.statement_identifier = camt053_get_str(d_stmt, "Id", name="Stmt.Id")

195 stm.statement_number = camt053_get_str(d_stmt, "LglSeqNb", name="Stmt.LglSeqNb")

196 stm.record_date = camt053_get_dt(d_stmt, "CreDtTm", name="Stmt.CreDtTm")

197 stm.begin_date = camt053_get_dt(d_frto, "FrDtTm", name="Stmt.FrDtTm").date()

198 stm.end_date = camt053_get_dt(d_frto, "ToDtTm", name="Stmt.ToDtTm").date()

199 stm.currency_code = camt053_get_str(d_acct, "Ccy", name="Stmt.Acct.Ccy")

200 if stm.currency_code != account.currency:

201 raise ValidationError(

202 _(

203 "Account currency {account_currency} does not match statement entry currency {statement_currency}".format(

204 statement_currency=stm.currency_code, account_currency=account.currency

205 )

206 )

207 )

208 stm.owner_name = camt053_get_str(d_ownr, "Nm", name="Stm.Acct.Ownr.Nm")

209 stm.begin_balance, stm.begin_balance_date = camt053_get_stmt_bal(d_stmt, "OPBD")

210 if stm.begin_balance_date is None:

211 stm.begin_balance_date = stm.begin_date

212 stm.record_count = camt053_get_int_or_none(d_txsummary.get("TtlNtries", {}), "NbOfNtries", name="Stmt.TxsSummry.TtlNtries.NbOfNtries") or 0

213 stm.bank_specific_info_1 = camt053_get_str(d_stmt, "AddtlStmtInf", required=False)[:1024]

214 for k, v in kw.items():

215 setattr(stm, k, v)

216 stm.full_clean()

217 stm.save()

218

219 e_deposit = EntryType.objects.filter(code=settings.E_BANK_DEPOSIT).first()

220 if not e_deposit:

221 raise ValidationError(_("entry.type.missing") + " ({}): {}".format("settings.E_BANK_DEPOSIT", settings.E_BANK_DEPOSIT))

222 assert isinstance(e_deposit, EntryType)

223 e_withdraw = EntryType.objects.filter(code=settings.E_BANK_WITHDRAW).first()

224 if not e_withdraw:

225 raise ValidationError(_("entry.type.missing") + " ({}): {}".format("settings.E_BANK_WITHDRAW", settings.E_BANK_WITHDRAW))

226 assert isinstance(e_withdraw, EntryType)

227 e_types = {

228 "CRDT": e_deposit,

229 "DBIT": e_withdraw,

230 }

231 record_type_map = {

232 "CRDT": "1",

233 "DBIT": "2",

234 }

235

236 for ntry in d_ntry:

237 archive_id = ntry.get("AcctSvcrRef", "")

238 amount, cur = camt053_get_currency(ntry, "Amt", name="Stmt.Ntry[{}].Amt".format(archive_id))

239 if cur != account.currency:

240 raise ValidationError(

241 _(

242 "Account currency {account_currency} does not match statement entry currency {statement_currency}".format(

243 statement_currency=cur, account_currency=account.currency

244 )

245 )

246 )

247

248 cdt_dbt_ind = ntry["CdtDbtInd"]

249 e_type = e_types.get(cdt_dbt_ind, None)

250 if not e_type:

251 raise ValidationError(_("Statement entry type {} not supported").format(cdt_dbt_ind))

252

253 rec = StatementRecord(statement=stm, account=account, type=e_type)

254 rec.amount = amount

255 rec.archive_identifier = archive_id

256 rec.entry_type = record_type_map[cdt_dbt_ind]

257 rec.record_date = record_date = camt053_get_date(ntry.get("BookgDt", {}), "Dt", name="Stmt.Ntry[{}].BkkgDt.Dt".format(archive_id))

258 rec.value_date = camt053_get_date(ntry.get("ValDt", {}), "Dt", name="Stmt.Ntry[{}].ValDt.Dt".format(archive_id))

259 rec.delivery_method = DELIVERY_FROM_BANK_SYSTEM

260

261 d_bktxcd = ntry.get("BkTxCd", {})

262 d_domn = d_bktxcd.get("Domn", {})

263 d_family = d_domn.get("Fmly", {})

264 d_prtry = d_bktxcd.get("Prtry", {})

265 rec.record_domain = record_domain = camt053_get_str(d_domn, "Cd", name="Stmt.Ntry[{}].BkTxCd.Domn.Cd".format(archive_id))

266 rec.record_code = camt053_domain_from_record_code(record_domain)

267 rec.family_code = camt053_get_str(d_family, "Cd", name="Stmt.Ntry[{}].BkTxCd.Domn.Family.Cd".format(archive_id))

268 rec.sub_family_code = camt053_get_str(d_family, "SubFmlyCd", name="Stmt.Ntry[{}].BkTxCd.Domn.Family.SubFmlyCd".format(archive_id))

269 rec.record_description = camt053_get_str(d_prtry, "Cd", required=False)

270

271 rec.full_clean()

272 rec.save()

273

274 for dtl_batch in ntry.get("NtryDtls", []):

275 batch_identifier = dtl_batch.get("Btch", {}).get("MsgId", "")

276 dtl_ix = 0

277 for dtl in dtl_batch.get("TxDtls", []):

278 d = StatementRecordDetail(record=rec, batch_identifier=batch_identifier)

279

280 d_amt_dtl = dtl.get("AmtDtls", {})

281 d_txamt = d_amt_dtl.get("TxAmt", {})

282 d_xchg = d_txamt.get("CcyXchg", None)

283

284 d.amount, d.currency_code = camt053_get_currency(d_txamt, "Amt", required=False)

285 d.instructed_amount, source_currency = camt053_get_currency(d_amt_dtl.get("InstdAmt", {}), "Amt", required=False)

286 if (not d_xchg and source_currency and source_currency != d.currency_code) or (d_xchg and not source_currency):

287 raise ValidationError(_("Inconsistent Stmt.Ntry[{}].NtryDtls.TxDtls[{}].AmtDtls".format(archive_id, dtl_ix)))

288

289 if source_currency and source_currency != d.currency_code:

290 source_currency = camt053_get_str(d_xchg, "SrcCcy", default=source_currency, required=False)

291 target_currency = camt053_get_str(d_xchg, "TrgCcy", default=d.currency_code, required=False)

292 unit_currency = camt053_get_str(d_xchg, "UnitCcy", default="", required=False)

293 exchange_rate_str = camt053_get_str(d_xchg, "XchgRate", default="", required=False)

294 exchange_rate = dec4(exchange_rate_str) if exchange_rate_str else None

295 exchange_source = CurrencyExchangeSource.objects.get_or_create(name=account_number)[0]

296 d.exchange = CurrencyExchange.objects.get_or_create(

297 record_date=record_date,

298 source_currency=source_currency,

299 target_currency=target_currency,

300 unit_currency=unit_currency,

301 exchange_rate=exchange_rate,

302 source=exchange_source,

303 )[0]

304

305 d_refs = dtl.get("Refs", {})

306 d.archive_identifier = d_refs.get("AcctSvcrRef", "")

307 d.end_to_end_identifier = d_refs.get("EndToEndId", "")

308

309 d_parties = dtl.get("RltdPties", {})

310 d_dbt = d_parties.get("Dbtr", {})

311 d.debtor_name = d_dbt.get("Nm", "")

312 d_udbt = d_parties.get("UltmtDbtr", {})

313 d.ultimate_debtor_name = d_udbt.get("Nm", "")

314 d_cdtr = d_parties.get("Cdtr", {})

315 d.creditor_name = d_cdtr.get("Nm", "")

316 d_cdtr_acct = d_parties.get("CdtrAcct", {})

317 d_cdtr_acct_id = d_cdtr_acct.get("Id", {})

318 d.creditor_account = d_cdtr_acct_id.get("IBAN", "")

319 if d.creditor_account:

320 d.creditor_account_scheme = "IBAN"

321 else:

322 d_cdtr_acct_id_othr = d_cdtr_acct_id.get("Othr") or {}

323 d.creditor_account_scheme = d_cdtr_acct_id_othr.get("SchmeNm", {}).get("Cd", "")

324 d.creditor_account = d_cdtr_acct_id_othr.get("Id") or ""

325

326 d_rmt = dtl.get("RmtInf", {})

327 for ustrd in d_rmt.get("Ustrd") or []:

328 if d.unstructured_remittance_info:

329 d.unstructured_remittance_info += "\n"

330 d.unstructured_remittance_info = str(ustrd).rstrip()

331

332 d_rltd_dts = dtl.get("RltdDts", {})

333 d.paid_date = camt053_get_dt(d_rltd_dts, "AccptncDtTm") if "AccptncDtTm" in d_rltd_dts else None

334

335 d.full_clean()

336 d.save()

337

338 st = StatementRecordRemittanceInfo(detail=d)

339 for strd in d_rmt.get("Strd", []):

340 additional_info = strd.get("AddtlRmtInf", "")

341 has_additional_info = bool(additional_info and st.additional_info)

342 amount, currency_code = camt053_get_currency(strd.get("RfrdDocAmt", {}), "RmtdAmt", required=False)

343 has_amount = bool(amount and st.amount)

344 reference = strd.get("CdtrRefInf", {}).get("Ref", "")

345 has_reference = bool(reference and st.reference)

346

347 # check if new remittance info record is needed

348 if has_additional_info or has_amount or has_reference:

349 st = StatementRecordRemittanceInfo(detail=d)

350

351 if additional_info:

352 st.additional_info = additional_info

353 if amount:

354 st.amount, st.currency_code = amount, currency_code

355 if reference:

356 st.reference = reference

357

358 st.full_clean()

359 st.save()

360

361 dtl_ix += 1

362

363 # fill record name from details

364 assert rec.type

365 if not rec.name:

366 if rec.type.code == e_withdraw.code:

367 rec.name = camt053_get_unified_str(rec.detail_set.all(), "creditor_name")

368 elif rec.type.code == e_deposit.code:

369 rec.name = camt053_get_unified_str(rec.detail_set.all(), "debtor_name")

370 if not rec.recipient_account_number:

371 rec.recipient_account_number = camt053_get_unified_str(rec.detail_set.all(), "creditor_account")

372 if not rec.remittance_info:

373 rec.remittance_info = camt053_get_unified_str(StatementRecordRemittanceInfo.objects.all().filter(detail__record=rec).order_by("id").distinct(), "reference")

374 if not rec.paid_date:

375 paid_date = camt053_get_unified_val(rec.detail_set.all(), "paid_date", default=None)

376 if paid_date:

377 assert isinstance(paid_date, datetime)

378 rec.paid_date = paid_date.date()

379

380 rec.full_clean()

381 rec.save()

382

383 return stm

384

385

386 def camt054_parse_file(filename: str) -> dict:

387 if parse_filename_suffix(filename).upper() not in CAMT054_STATEMENT_SUFFIXES:

388 raise ValidationError(

389 _('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format(

390 filename=filename, suffixes=", ".join(CAMT054_STATEMENT_SUFFIXES), file_type="camt.054"

391 )

392 )

393 with open(filename, "rb") as fp:

394 data = xml_to_dict(fp.read(), array_tags=CAMT054_ARRAY_TAGS, int_tags=CAMT054_INT_TAGS)

395 return data

396

397

398 def camt054_parse_ntfctn_acct(ntfctn: dict, default_currency: str = "EUR") -> Tuple[str, str]:

399 """

400 Returns account_number, currency from Ntfctn

401 """

402 acc_data = ntfctn["Acct"]

403 currency = acc_data.get("Ccy") or default_currency

404 account_number = acc_data["Id"].get("IBAN") or acc_data["Id"].get("BBAN")

405 return account_number, currency

406

407

408 def camt054_parse_date(data: dict, key: str) -> date:

409 val = data.get(key)

410 if not val or "Dt" not in val:

411 raise Exception(_("Failed to parse date '{}'").format(key))

412 return parse_date(val["Dt"])

413

414

415 def camt054_parse_amt(data: dict, key: str) -> Tuple[Decimal, str]:

416 val = data.get(key) or {}

417 amt = val.get("Amt")

418 if not amt or "@" not in amt or "@Ccy" not in amt:

419 raise Exception(_("Failed to parse amount '{}'").format(key))

420 return Decimal(amt["@"]), amt["@Ccy"]

421

422

423 def camt054_parse_rmtinf(data: dict, key: str) -> str:

424 out = ""

425 val = data.get(key) or {}

426 ustrd_list = val.get("Ustrd")

427 if ustrd_list:

428 out = "\n".join(str(ustrd).rstrip() for ustrd in ustrd_list)

429 strd_list = val.get("Strd") or []

430 for strd in strd_list:

431 cdtrrefinf = strd.get("CdtrRefInf") or {}

432 ref_val = cdtrrefinf.get("Ref") or ""

433 if ref_val:

434 if out:

435 out += "\n"

436 out += ref_val

437 return out

438

439

440 def camt054_parse_dbtr(data: dict, key: str) -> str:

441 val = data.get(key) or {}

442 if not val:

443 return ""

444 dbtr = val.get("Dbtr")

445 if not dbtr or "Nm" not in dbtr:

446 raise Exception(_("Failed to parse debtor '{}'").format(key))

447 return dbtr["Nm"]

448

449

450 def camt054_parse_rltdagts_cdtragt_fininstnid_bic(data: dict, key: str) -> str:

451 rltdagts = data.get(key) or {}

452 cdtragt = rltdagts.get("CdtrAgt") or {}

453 fininstnid = cdtragt.get("FinInstnId") or {}

454 return fininstnid.get("BIC") or ""

455

456

457 def camt054_parse_refs_endtoendid(data: dict, key: str) -> str:

458 refs = data.get(key) or {}

459 return refs.get("EndToEndId") or ""

460

461

462@transaction.atomic

463 def camt054_create_reference_payment_batch(ntfctn: dict, name: str, file: ReferencePaymentBatchFile) -> ReferencePaymentBatch:

464 account_number, account_currency = camt054_parse_ntfctn_acct(ntfctn)

465 account = Account.objects.get(name=account_number, currency=account_currency)

466 assert isinstance(account, Account)

467 created_datetime = parse_datetime(ntfctn["CreDtTm"])

468 identifier = ntfctn["Id"]

469 batch = ReferencePaymentBatch(record_date=created_datetime, file=file, identifier=identifier, name=name)

470 batch.clean()

471 batch.save()

472 e_deposit = EntryType.objects.get(code=settings.E_BANK_DEPOSIT)

473 e_withdraw = EntryType.objects.get(code=settings.E_BANK_WITHDRAW)

474 for ntry in ntfctn["Ntry"]:

475 rec = ReferencePaymentRecord(batch=batch, account_number=account_number, account=account)

476 rec.record_date = camt054_parse_date(ntry, "BookgDt")

477 rec.record_type = cdtdbtind = ntry.get("CdtDbtInd") or ""

478 if cdtdbtind == "CRDT":

479 rec.type = e_deposit

480 elif cdtdbtind == "DBIT":

481 rec.type = e_withdraw

482 else:

483 raise Exception(_("Unknown credit/debit indicator '{}'").format(cdtdbtind))

484 rec.value_date = camt054_parse_date(ntry, "ValDt")

485 ntrydtls_list = ntry.get("NtryDtls") or []

486 for ntrydtls0 in ntrydtls_list:

487 rec_tx = clone_model(rec)

488 assert isinstance(rec_tx, ReferencePaymentRecord)

489 txdtls = ntrydtls0["TxDtls"]

490 amtdtls = txdtls["AmtDtls"]

491 rec_tx.instructed_amount, rec_tx.instructed_currency = camt054_parse_amt(amtdtls, "InstdAmt")

492 rec_tx.amount, rec_currency = camt054_parse_amt(amtdtls, "TxAmt")

493 if rec_currency != account_currency:

494 raise Exception(_("Account currency {} does not match record currency {}").format(account_currency, rec_currency))

495 rec_tx.remittance_info = camt054_parse_rmtinf(txdtls, "RmtInf")

496 rec_tx.payer_name = camt054_parse_dbtr(txdtls, "RltdPties")

497 rec_tx.creditor_bank_bic = camt054_parse_rltdagts_cdtragt_fininstnid_bic(txdtls, "RltdAgts")

498 rec_tx.end_to_end_identifier = camt054_parse_refs_endtoendid(txdtls, "Refs")

499 rec_tx.clean()

500 rec_tx.save()

501 return batch