Coverage for jbank/camt.py: 85%

# Coverage for jbank/camt.py : 85%

## 253 statements

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1 from datetime import datetime, date

2 from decimal import Decimal

3 from typing import Tuple, Any, Optional, Dict

4

5 from django.conf import settings

6 from django.core.exceptions import ValidationError

7 from django.db import transaction

8 from django.utils.dateparse import parse_date

9 from django.utils.translation import gettext as _

10 from jacc.models import Account, EntryType

11 from jutil.format import dec2, dec4

12 from jutil.parse import parse_datetime

13 from jbank.models import (

14 StatementFile,

15 Statement,

16 StatementRecord,

17 DELIVERY_FROM_BANK_SYSTEM,

18 StatementRecordDetail,

19 CurrencyExchange,

20 StatementRecordRemittanceInfo,

21 CurrencyExchangeSource,

22)

23 from jbank.parsers import parse_filename_suffix

24 from jutil.xml import xml_to_dict

25

26

27 CAMT053_STATEMENT_SUFFIXES = ("XML", "XT", "CAMT")

28

29 CAMT053_ARRAY_TAGS = ["Bal", "Ntry", "NtryDtls", "TxDtls", "Strd"]

30

31 CAMT053_INT_TAGS = ["NbOfNtries", "NbOfTxs"]

32

33

34 def camt053_get_iban(data: dict) -> str:

35 return data.get("BkToCstmrStmt", {}).get("Stmt", {}).get("Acct", {}).get("Id", {}).get("IBAN", "")

36

37

38 def camt053_get_val(data: dict, key: str, default: Any = None, required: bool = True, name: str = "") -> Any:

39 if key not in data:

40 if required:

41 raise ValidationError(_("camt.053 field {} missing").format(name if name else key))

42 return default

43 return data[key]

44

45

46 def camt053_get_str(data: dict, key: str, default: str = "", required: bool = True, name: str = "") -> str:

47 return str(camt053_get_val(data, key, default, required, name))

48

49

50 def camt053_get_currency(data: dict, key: str, required: bool = True, name: str = "") -> Tuple[Optional[Decimal], str]:

51 try:

52 v = camt053_get_val(data, key, default=None, required=False, name=name)

53 if v is not None:

54 amount = dec2(v["@"])

55 currency_code = v["@Ccy"]

56 return amount, currency_code

57 except Exception:

58 pass

59 if required:

60 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "currency"))

61 return None, ""

62

63

64 def camt053_get_dt(data: Dict[str, Any], key: str, name: str = "") -> datetime:

65 s = camt053_get_val(data, key, None, True, name)

66 val = parse_datetime(s)

67 if val is None:

68 raise ValidationError(

69 _("camt.053 field {} type {} missing or invalid").format(name, "datetime") + ": {}".format(s)

70 )

71 return val

72

73

74 def camt053_get_int(data: Dict[str, Any], key: str, name: str = "") -> int:

75 s = camt053_get_val(data, key, None, True, name)

76 try:

77 return int(s)

78 except Exception:

79 pass

80 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "int"))

81

82

83 def camt053_get_date(

84 data: dict, key: str, default: Optional[date] = None, required: bool = True, name: str = ""

85) -> date:

86 s = camt053_get_val(data, key, default, required, name)

87 try:

88 val = parse_date(s[:10])

89 if val is None:

90 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "date"))

91 assert isinstance(val, date)

92 return val

93 except Exception:

94 pass

95 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format(name, "date") + ": {}".format(s))

96

97

98 def camt053_parse_statement_from_file(filename: str) -> dict:

99 if parse_filename_suffix(filename).upper() not in CAMT053_STATEMENT_SUFFIXES:

100 raise ValidationError(

101 _('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format(

102 filename=filename, suffixes=", ".join(CAMT053_STATEMENT_SUFFIXES), file_type="camt.053"

103 )

104 )

105 with open(filename, "rb") as fp:

106 data = xml_to_dict(fp.read(), array_tags=CAMT053_ARRAY_TAGS, int_tags=CAMT053_INT_TAGS)

107 return data

108

109

110 def camt053_get_stmt_bal(d_stmt: dict, bal_type: str) -> Tuple[Decimal, Optional[date]]:

111 for bal in d_stmt.get("Bal", []):

112 if bal.get("Tp", {}).get("CdOrPrtry", {}).get("Cd", "") == bal_type:

113 amt = Decimal(bal.get("Amt", {}).get("@", ""))

114 dt_data = bal.get("Dt", {})

115 dt = None

116 if "Dt" in dt_data:

117 dt = camt053_get_date(dt_data, "Dt", name="Stmt.Bal[{}].Dt.Dt".format(bal_type))

118 return amt, dt

119 raise ValidationError(_("camt.053 field {} type {} missing or invalid").format("Stmt.Bal.Tp.CdOrPrty.Cd", bal_type))

120

121

122 def camt053_domain_from_record_code(record_domain: str) -> str:

123 if record_domain == "PMNT":

124 return "700"

125 if record_domain == "LDAS":

126 return "761"

127 return ""

128

129

130 def camt053_get_unified_val(qs, k: str, default: Any) -> Any:

131 v = default

132 for e in qs:

133 v2 = getattr(e, k)

134 if v == default:

135 v = v2

136 elif v and v2 and v2 != v:

137 return default

138 return v

139

140

141 def camt053_get_unified_str(qs, k: str) -> str:

142 return camt053_get_unified_val(qs, k, "")

143

144

145@transaction.atomic # noqa

146 def camt053_create_statement(statement_data: dict, name: str, file: StatementFile, **kw) -> Statement: # noqa

147 """

148 Creates camt.053 Statement from statement data parsed by camt053_parse_statement_from_file()

149 :param statement_data: XML data in form of dict

150 :param name: File name of the account statement

151 :param file: Source statement file

152 :return: Statement

153 """

154 account_number = camt053_get_iban(statement_data)

155 if not account_number:

156 raise ValidationError("{name}: ".format(name=name) + _("account.not.found").format(account_number=""))

157 accounts = list(Account.objects.filter(name=account_number))

158 if len(accounts) != 1:

159 raise ValidationError(

160 "{name}: ".format(name=name) + _("account.not.found").format(account_number=account_number)

161 )

162 account = accounts[0]

163 assert isinstance(account, Account)

164

165 d_stmt = statement_data.get("BkToCstmrStmt", {}).get("Stmt", {})

166 d_acct = d_stmt.get("Acct", {})

167 d_ownr = d_acct.get("Ownr", {})

168 d_ntry = d_stmt.get("Ntry", [])

169 d_frto = d_stmt.get("FrToDt", {})

170 d_txsummary = d_stmt.get("TxsSummry", {})

171

172 if Statement.objects.filter(name=name, account=account).first():

173 raise ValidationError("Bank account {} statement {} of processed already".format(account_number, name))

174 stm = Statement(name=name, account=account, file=file)

175 stm.account_number = stm.iban = account_number

176 stm.bic = camt053_get_str(d_acct.get("Svcr", {}).get("FinInstnId", {}), "BIC", name="Stmt.Acct.Svcr.FinInstnId.BIC")

177 stm.statement_identifier = camt053_get_str(d_stmt, "Id", name="Stmt.Id")

178 stm.statement_number = camt053_get_str(d_stmt, "LglSeqNb", name="Stmt.LglSeqNb")

179 stm.record_date = camt053_get_dt(d_stmt, "CreDtTm", name="Stmt.CreDtTm")

180 stm.begin_date = camt053_get_dt(d_frto, "FrDtTm", name="Stmt.FrDtTm").date()

181 stm.end_date = camt053_get_dt(d_frto, "ToDtTm", name="Stmt.ToDtTm").date()

182 stm.currency_code = camt053_get_str(d_acct, "Ccy", name="Stmt.Acct.Ccy")

183 if stm.currency_code != account.currency:

184 raise ValidationError(

185 _(

186 "Account currency {account_currency} does not match statement entry currency {statement_currency}".format(

187 statement_currency=stm.currency_code, account_currency=account.currency

188 )

189 )

190 )

191 stm.owner_name = camt053_get_str(d_ownr, "Nm", name="Stm.Acct.Ownr.Nm")

192 stm.begin_balance, stm.begin_balance_date = camt053_get_stmt_bal(d_stmt, "OPBD")

193 if stm.begin_balance_date is None:

194 stm.begin_balance_date = stm.begin_date

195 stm.record_count = camt053_get_int(

196 d_txsummary.get("TtlNtries", {}), "NbOfNtries", name="Stmt.TxsSummry.TtlNtries.NbOfNtries"

197 )

198 stm.bank_specific_info_1 = camt053_get_str(d_stmt, "AddtlStmtInf", required=False)[:1024]

199 for k, v in kw.items():

200 setattr(stm, k, v)

201 stm.full_clean()

202 stm.save()

203

204 e_deposit = EntryType.objects.filter(code=settings.E_BANK_DEPOSIT).first()

205 if not e_deposit:

206 raise ValidationError(

207 _("entry.type.missing") + " ({}): {}".format("settings.E_BANK_DEPOSIT", settings.E_BANK_DEPOSIT)

208 )

209 assert isinstance(e_deposit, EntryType)

210 e_withdraw = EntryType.objects.filter(code=settings.E_BANK_WITHDRAW).first()

211 if not e_withdraw:

212 raise ValidationError(

213 _("entry.type.missing") + " ({}): {}".format("settings.E_BANK_WITHDRAW", settings.E_BANK_WITHDRAW)

214 )

215 assert isinstance(e_withdraw, EntryType)

216 e_types = {

217 "CRDT": e_deposit,

218 "DBIT": e_withdraw,

219 }

220 record_type_map = {

221 "CRDT": "1",

222 "DBIT": "2",

223 }

224

225 for ntry in d_ntry:

226 archive_id = ntry.get("AcctSvcrRef", "")

227 amount, cur = camt053_get_currency(ntry, "Amt", name="Stmt.Ntry[{}].Amt".format(archive_id))

228 if cur != account.currency:

229 raise ValidationError(

230 _(

231 "Account currency {account_currency} does not match statement entry currency {statement_currency}".format(

232 statement_currency=cur, account_currency=account.currency

233 )

234 )

235 )

236

237 cdt_dbt_ind = ntry["CdtDbtInd"]

238 e_type = e_types.get(cdt_dbt_ind, None)

239 if not e_type:

240 raise ValidationError(_("Statement entry type {} not supported").format(cdt_dbt_ind))

241

242 rec = StatementRecord(statement=stm, account=account, type=e_type)

243 rec.amount = amount

244 rec.archive_identifier = archive_id

245 rec.entry_type = record_type_map[cdt_dbt_ind]

246 rec.record_date = record_date = camt053_get_date(

247 ntry.get("BookgDt", {}), "Dt", name="Stmt.Ntry[{}].BkkgDt.Dt".format(archive_id)

248 )

249 rec.value_date = camt053_get_date(ntry.get("ValDt", {}), "Dt", name="Stmt.Ntry[{}].ValDt.Dt".format(archive_id))

250 rec.delivery_method = DELIVERY_FROM_BANK_SYSTEM

251

252 d_bktxcd = ntry.get("BkTxCd", {})

253 d_domn = d_bktxcd.get("Domn", {})

254 d_family = d_domn.get("Fmly", {})

255 d_prtry = d_bktxcd.get("Prtry", {})

256 rec.record_domain = record_domain = camt053_get_str(

257 d_domn, "Cd", name="Stmt.Ntry[{}].BkTxCd.Domn.Cd".format(archive_id)

258 )

259 rec.record_code = camt053_domain_from_record_code(record_domain)

260 rec.family_code = camt053_get_str(d_family, "Cd", name="Stmt.Ntry[{}].BkTxCd.Domn.Family.Cd".format(archive_id))

261 rec.sub_family_code = camt053_get_str(

262 d_family, "SubFmlyCd", name="Stmt.Ntry[{}].BkTxCd.Domn.Family.SubFmlyCd".format(archive_id)

263 )

264 rec.record_description = camt053_get_str(d_prtry, "Cd", required=False)

265

266 rec.full_clean()

267 rec.save()

268

269 for dtl_batch in ntry.get("NtryDtls", []):

270 batch_identifier = dtl_batch.get("Btch", {}).get("MsgId", "")

271 dtl_ix = 0

272 for dtl in dtl_batch.get("TxDtls", []):

273 d = StatementRecordDetail(record=rec, batch_identifier=batch_identifier)

274

275 d_amt_dtl = dtl.get("AmtDtls", {})

276 d_txamt = d_amt_dtl.get("TxAmt", {})

277 d_xchg = d_txamt.get("CcyXchg", None)

278

279 d.amount, d.currency_code = camt053_get_currency(d_txamt, "Amt", required=False)

280 d.instructed_amount, source_currency = camt053_get_currency(

281 d_amt_dtl.get("InstdAmt", {}), "Amt", required=False

282 )

283 if (not d_xchg and source_currency and source_currency != d.currency_code) or (

284 d_xchg and not source_currency

285 ):

286 raise ValidationError(

287 _("Inconsistent Stmt.Ntry[{}].NtryDtls.TxDtls[{}].AmtDtls".format(archive_id, dtl_ix))

288 )

289

290 if source_currency and source_currency != d.currency_code:

291 source_currency = camt053_get_str(d_xchg, "SrcCcy", default=source_currency, required=False)

292 target_currency = camt053_get_str(d_xchg, "TrgCcy", default=d.currency_code, required=False)

293 unit_currency = camt053_get_str(d_xchg, "UnitCcy", default="", required=False)

294 exchange_rate_str = camt053_get_str(d_xchg, "XchgRate", default="", required=False)

295 exchange_rate = dec4(exchange_rate_str) if exchange_rate_str else None

296 exchange_source = CurrencyExchangeSource.objects.get_or_create(name=account_number)[0]

297 d.exchange = CurrencyExchange.objects.get_or_create(

298 record_date=record_date,

299 source_currency=source_currency,

300 target_currency=target_currency,

301 unit_currency=unit_currency,

302 exchange_rate=exchange_rate,

303 source=exchange_source,

304 )[0]

305

306 d_refs = dtl.get("Refs", {})

307 d.archive_identifier = d_refs.get("AcctSvcrRef", "")

308 d.end_to_end_identifier = d_refs.get("EndToEndId", "")

309

310 d_parties = dtl.get("RltdPties", {})

311 d_dbt = d_parties.get("Dbtr", {})

312 d.debtor_name = d_dbt.get("Nm", "")

313 d_udbt = d_parties.get("UltmtDbtr", {})

314 d.ultimate_debtor_name = d_udbt.get("Nm", "")

315 d_cdtr = d_parties.get("Cdtr", {})

316 d.creditor_name = d_cdtr.get("Nm", "")

317 d_cdtr_acct = d_parties.get("CdtrAcct", {})

318 d_cdtr_acct_id = d_cdtr_acct.get("Id", {})

319 d.creditor_account = d_cdtr_acct_id.get("IBAN", "")

320 if d.creditor_account:

321 d.creditor_account_scheme = "IBAN"

322 else:

323 d_cdtr_acct_id_othr = d_cdtr_acct_id.get("Othr") or {}

324 d.creditor_account_scheme = d_cdtr_acct_id_othr.get("SchmeNm", {}).get("Cd", "")

325 d.creditor_account = d_cdtr_acct_id_othr.get("Id") or ""

326

327 d_rmt = dtl.get("RmtInf", {})

328 d.unstructured_remittance_info = d_rmt.get("Ustrd", "")

329

330 d_rltd_dts = dtl.get("RltdDts", {})

331 d.paid_date = camt053_get_dt(d_rltd_dts, "AccptncDtTm") if "AccptncDtTm" in d_rltd_dts else None

332

333 d.full_clean()

334 d.save()

335

336 st = StatementRecordRemittanceInfo(detail=d)

337 for strd in d_rmt.get("Strd", []):

338 additional_info = strd.get("AddtlRmtInf", "")

339 has_additional_info = bool(additional_info and st.additional_info)

340 amount, currency_code = camt053_get_currency(strd.get("RfrdDocAmt", {}), "RmtdAmt", required=False)

341 has_amount = bool(amount and st.amount)

342 reference = strd.get("CdtrRefInf", {}).get("Ref", "")

343 has_reference = bool(reference and st.reference)

344

345 # check if new remittance info record is needed

346 if has_additional_info or has_amount or has_reference:

347 st = StatementRecordRemittanceInfo(detail=d)

348

349 if additional_info:

350 st.additional_info = additional_info

351 if amount:

352 st.amount, st.currency_code = amount, currency_code

353 if reference:

354 st.reference = reference

355

356 st.full_clean()

357 st.save()

358

359 dtl_ix += 1

360

361 # fill record name from details

362 assert rec.type

363 if not rec.name:

364 if rec.type.code == e_withdraw.code:

365 rec.name = camt053_get_unified_str(rec.detail_set.all(), "creditor_name")

366 elif rec.type.code == e_deposit.code:

367 rec.name = camt053_get_unified_str(rec.detail_set.all(), "debtor_name")

368 if not rec.recipient_account_number:

369 rec.recipient_account_number = camt053_get_unified_str(rec.detail_set.all(), "creditor_account")

370 if not rec.remittance_info:

371 rec.remittance_info = camt053_get_unified_str(

372 StatementRecordRemittanceInfo.objects.all().filter(detail__record=rec), "reference"

373 )

374 if not rec.paid_date:

375 paid_date = camt053_get_unified_val(rec.detail_set.all(), "paid_date", default=None)

376 if paid_date:

377 assert isinstance(paid_date, datetime)

378 rec.paid_date = paid_date.date()

379

380 rec.full_clean()

381 rec.save()

382

383 return stm