Coverage for jbank/camt.py: 84%

# Coverage for jbank/camt.py : 84%

## 244 statements

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1 from datetime import datetime, date

2 from decimal import Decimal

3 from typing import Tuple, Any, Optional, Dict

4

5 from django.conf import settings

6 from django.core.exceptions import ValidationError

7 from django.db import transaction

8 from django.utils.dateparse import parse_date

9 from django.utils.translation import gettext as _

10 from jacc.models import Account, EntryType

11 from jutil.format import dec2, dec4

12 from jutil.parse import parse_datetime

13 from jbank.models import StatementFile, Statement, StatementRecord, DELIVERY_FROM_BANK_SYSTEM, \

14 StatementRecordDetail, CurrencyExchange, StatementRecordRemittanceInfo, CurrencyExchangeSource

15 from jbank.parsers import parse_filename_suffix

16 from jutil.xml import xml_to_dict

17

18

19 CAMT053_STATEMENT_SUFFIXES = ('XML', 'XT', 'CAMT')

20

21 CAMT053_ARRAY_TAGS = ['Bal', 'Ntry', 'NtryDtls', 'TxDtls', 'Strd']

22

23 CAMT053_INT_TAGS = ['NbOfNtries', 'NbOfTxs']

24

25

26 def camt053_get_iban(data: dict) -> str:

27 return data.get('BkToCstmrStmt', {}).get('Stmt', {}).get('Acct', {}).get('Id', {}).get('IBAN', '')

28

29

30 def camt053_get_val(data: dict, key: str, default: Any = None, required: bool = True, name: str = '') -> Any:

31 if key not in data:

32 if required:

33 raise ValidationError(_('camt.053 field {} missing').format(name if name else key))

34 return default

35 return data[key]

36

37

38 def camt053_get_str(data: dict, key: str, default: str = '', required: bool = True, name: str = '') -> str:

39 return str(camt053_get_val(data, key, default, required, name))

40

41

42 def camt053_get_currency(data: dict, key: str, required: bool = True,

43 name: str = '') -> Tuple[Optional[Decimal], str]:

44 try:

45 v = camt053_get_val(data, key, default=None, required=False, name=name)

46 if v is not None:

47 amount = dec2(v['@'])

48 currency_code = v['@Ccy']

49 return amount, currency_code

50 except Exception:

51 pass

52 if required:

53 raise ValidationError(_('camt.053 field {} type {} missing or invalid').format(name, 'currency'))

54 return None, ''

55

56

57 def camt053_get_dt(data: Dict[str, Any], key: str, name: str = '') -> datetime:

58 s = camt053_get_val(data, key, None, True, name)

59 val = parse_datetime(s)

60 if val is None:

61 raise ValidationError(_('camt.053 field {} type {} missing or invalid').format(name, 'datetime') + ': {}'.format(s))

62 return val

63

64

65 def camt053_get_int(data: Dict[str, Any], key: str, name: str = '') -> int:

66 s = camt053_get_val(data, key, None, True, name)

67 try:

68 return int(s)

69 except Exception:

70 pass

71 raise ValidationError(_('camt.053 field {} type {} missing or invalid').format(name, 'int'))

72

73

74 def camt053_get_date(data: dict, key: str, default: Optional[date] = None, required: bool = True, name: str = '') -> date:

75 s = camt053_get_val(data, key, default, required, name)

76 try:

77 val = parse_date(s[:10])

78 if val is None:

79 raise ValidationError(_('camt.053 field {} type {} missing or invalid').format(name, 'date'))

80 assert isinstance(val, date)

81 return val

82 except Exception:

83 pass

84 raise ValidationError(_('camt.053 field {} type {} missing or invalid').format(name, 'date') + ': {}'.format(s))

85

86

87 def camt053_parse_statement_from_file(filename: str) -> dict:

88 if parse_filename_suffix(filename).upper() not in CAMT053_STATEMENT_SUFFIXES:

89 raise ValidationError(_('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format(

90 filename=filename, suffixes=', '.join(CAMT053_STATEMENT_SUFFIXES), file_type='camt.053'))

91 with open(filename, 'rb') as fp:

92 data = xml_to_dict(fp.read(), array_tags=CAMT053_ARRAY_TAGS, int_tags=CAMT053_INT_TAGS)

93 return data

94

95

96 def camt053_get_stmt_bal(d_stmt: dict, bal_type: str) -> Tuple[Decimal, Optional[date]]:

97 for bal in d_stmt.get('Bal', []):

98 if bal.get('Tp', {}).get('CdOrPrtry', {}).get('Cd', '') == bal_type:

99 amt = Decimal(bal.get('Amt', {}).get('@', ''))

100 dt_data = bal.get('Dt', {})

101 dt = None

102 if 'Dt' in dt_data:

103 dt = camt053_get_date(dt_data, 'Dt', name='Stmt.Bal[{}].Dt.Dt'.format(bal_type))

104 return amt, dt

105 raise ValidationError(_('camt.053 field {} type {} missing or invalid').format('Stmt.Bal.Tp.CdOrPrty.Cd', bal_type))

106

107

108 def camt053_domain_from_record_code(record_domain: str) -> str:

109 if record_domain == 'PMNT':

110 return '700'

111 if record_domain == 'LDAS':

112 return '761'

113 return ''

114

115

116 def camt053_get_unified_val(qs, k: str, default: Any) -> Any:

117 v = default

118 for e in qs:

119 v2 = getattr(e, k)

120 if v == default:

121 v = v2

122 elif v and v2 and v2 != v:

123 return default

124 return v

125

126

127 def camt053_get_unified_str(qs, k: str) -> str:

128 return camt053_get_unified_val(qs, k, '')

129

130

131@transaction.atomic # noqa

132 def camt053_create_statement(statement_data: dict, # noqa

133 name: str, file: StatementFile, **kw) -> Statement:

134 """

135 Creates camt.053 Statement from statement data parsed by camt053_parse_statement_from_file()

136 :param statement_data: XML data in form of dict

137 :param name: File name of the account statement

138 :param file: Source statement file

139 :return: Statement

140 """

141 account_number = camt053_get_iban(statement_data)

142 if not account_number:

143 raise ValidationError('{name}: '.format(name=name) + _("account.not.found").format(account_number=''))

144 accounts = list(Account.objects.filter(name=account_number))

145 if len(accounts) != 1:

146 raise ValidationError('{name}: '.format(name=name) + _("account.not.found").format(account_number=account_number))

147 account = accounts[0]

148 assert isinstance(account, Account)

149

150 d_stmt = statement_data.get('BkToCstmrStmt', {}).get('Stmt', {})

151 d_acct = d_stmt.get('Acct', {})

152 d_ownr = d_acct.get('Ownr', {})

153 d_ntry = d_stmt.get('Ntry', [])

154 d_frto = d_stmt.get('FrToDt', {})

155 d_txsummary = d_stmt.get('TxsSummry', {})

156

157 if Statement.objects.filter(name=name, account=account).first():

158 raise ValidationError('Bank account {} statement {} of processed already'.format(account_number, name))

159 stm = Statement(name=name, account=account, file=file)

160 stm.account_number = stm.iban = account_number

161 stm.bic = camt053_get_str(d_acct.get('Svcr', {}).get('FinInstnId', {}), 'BIC', name='Stmt.Acct.Svcr.FinInstnId.BIC')

162 stm.statement_identifier = camt053_get_str(d_stmt, 'Id', name='Stmt.Id')

163 stm.statement_number = camt053_get_str(d_stmt, 'LglSeqNb', name='Stmt.LglSeqNb')

164 stm.record_date = camt053_get_dt(d_stmt, 'CreDtTm', name='Stmt.CreDtTm')

165 stm.begin_date = camt053_get_dt(d_frto, 'FrDtTm', name='Stmt.FrDtTm').date()

166 stm.end_date = camt053_get_dt(d_frto, 'ToDtTm', name='Stmt.ToDtTm').date()

167 stm.currency_code = camt053_get_str(d_acct, 'Ccy', name='Stmt.Acct.Ccy')

168 if stm.currency_code != account.currency:

169 raise ValidationError(_('Account currency {account_currency} does not match statement entry currency {statement_currency}'.format(

170 statement_currency=stm.currency_code, account_currency=account.currency)))

171 stm.owner_name = camt053_get_str(d_ownr, 'Nm', name='Stm.Acct.Ownr.Nm')

172 stm.begin_balance, stm.begin_balance_date = camt053_get_stmt_bal(d_stmt, 'OPBD')

173 if stm.begin_balance_date is None:

174 stm.begin_balance_date = stm.begin_date

175 stm.record_count = camt053_get_int(d_txsummary.get('TtlNtries', {}), 'NbOfNtries', name='Stmt.TxsSummry.TtlNtries.NbOfNtries')

176 stm.bank_specific_info_1 = camt053_get_str(d_stmt, 'AddtlStmtInf', required=False)[:1024]

177 for k, v in kw.items():

178 setattr(stm, k, v)

179 stm.full_clean()

180 stm.save()

181

182 e_deposit = EntryType.objects.filter(code=settings.E_BANK_DEPOSIT).first()

183 if not e_deposit:

184 raise ValidationError(_('entry.type.missing') + ' ({}): {}'.format('settings.E_BANK_DEPOSIT', settings.E_BANK_DEPOSIT))

185 assert isinstance(e_deposit, EntryType)

186 e_withdraw = EntryType.objects.filter(code=settings.E_BANK_WITHDRAW).first()

187 if not e_withdraw:

188 raise ValidationError(_('entry.type.missing') + ' ({}): {}'.format('settings.E_BANK_WITHDRAW', settings.E_BANK_WITHDRAW))

189 assert isinstance(e_withdraw, EntryType)

190 e_types = {

191 'CRDT': e_deposit,

192 'DBIT': e_withdraw,

193 }

194 record_type_map = {

195 'CRDT': '1',

196 'DBIT': '2',

197 }

198

199 for ntry in d_ntry:

200 archive_id = ntry.get('AcctSvcrRef', '')

201 amount, cur = camt053_get_currency(ntry, 'Amt', name='Stmt.Ntry[{}].Amt'.format(archive_id))

202 if cur != account.currency:

203 raise ValidationError(_('Account currency {account_currency} does not match statement entry currency {statement_currency}'.format(

204 statement_currency=cur, account_currency=account.currency)))

205

206 cdt_dbt_ind = ntry['CdtDbtInd']

207 e_type = e_types.get(cdt_dbt_ind, None)

208 if not e_type:

209 raise ValidationError(_('Statement entry type {} not supported').format(cdt_dbt_ind))

210

211 rec = StatementRecord(statement=stm, account=account, type=e_type)

212 rec.amount = amount

213 rec.archive_identifier = archive_id

214 rec.entry_type = record_type_map[cdt_dbt_ind]

215 rec.record_date = record_date = camt053_get_date(ntry.get('BookgDt', {}), 'Dt', name='Stmt.Ntry[{}].BkkgDt.Dt'.format(archive_id))

216 rec.value_date = camt053_get_date(ntry.get('ValDt', {}), 'Dt', name='Stmt.Ntry[{}].ValDt.Dt'.format(archive_id))

217 rec.delivery_method = DELIVERY_FROM_BANK_SYSTEM

218

219 d_bktxcd = ntry.get('BkTxCd', {})

220 d_domn = d_bktxcd.get('Domn', {})

221 d_family = d_domn.get('Fmly', {})

222 d_prtry = d_bktxcd.get('Prtry', {})

223 rec.record_domain = record_domain = camt053_get_str(d_domn, 'Cd', name='Stmt.Ntry[{}].BkTxCd.Domn.Cd'.format(archive_id))

224 rec.record_code = camt053_domain_from_record_code(record_domain)

225 rec.family_code = camt053_get_str(d_family, 'Cd', name='Stmt.Ntry[{}].BkTxCd.Domn.Family.Cd'.format(archive_id))

226 rec.sub_family_code = camt053_get_str(d_family, 'SubFmlyCd', name='Stmt.Ntry[{}].BkTxCd.Domn.Family.SubFmlyCd'.format(archive_id))

227 rec.record_description = camt053_get_str(d_prtry, 'Cd', required=False)

228

229 rec.full_clean()

230 rec.save()

231

232 for dtl_batch in ntry.get('NtryDtls', []):

233 batch_identifier = dtl_batch.get('Btch', {}).get('MsgId', '')

234 dtl_ix = 0

235 for dtl in dtl_batch.get('TxDtls', []):

236 d = StatementRecordDetail(record=rec, batch_identifier=batch_identifier)

237

238 d_amt_dtl = dtl.get('AmtDtls', {})

239 d_txamt = d_amt_dtl.get('TxAmt', {})

240 d_xchg = d_txamt.get('CcyXchg', None)

241

242 d.amount, d.currency_code = camt053_get_currency(d_txamt, 'Amt', required=False)

243 d.instructed_amount, source_currency = camt053_get_currency(d_amt_dtl.get('InstdAmt', {}), 'Amt', required=False)

244 if (not d_xchg and source_currency and source_currency != d.currency_code) or (d_xchg and not source_currency):

245 raise ValidationError(_('Inconsistent Stmt.Ntry[{}].NtryDtls.TxDtls[{}].AmtDtls'.format(archive_id, dtl_ix)))

246

247 if source_currency and source_currency != d.currency_code:

248 source_currency = camt053_get_str(d_xchg, 'SrcCcy', default=source_currency, required=False)

249 target_currency = camt053_get_str(d_xchg, 'TrgCcy', default=d.currency_code, required=False)

250 unit_currency = camt053_get_str(d_xchg, 'UnitCcy', default='', required=False)

251 exchange_rate_str = camt053_get_str(d_xchg, 'XchgRate', default='', required=False)

252 exchange_rate = dec4(exchange_rate_str) if exchange_rate_str else None

253 exchange_source = CurrencyExchangeSource.objects.get_or_create(name=account_number)[0]

254 d.exchange = CurrencyExchange.objects.get_or_create(record_date=record_date, source_currency=source_currency,

255 target_currency=target_currency, unit_currency=unit_currency,

256 exchange_rate=exchange_rate, source=exchange_source)[0]

257

258 d_refs = dtl.get('Refs', {})

259 d.archive_identifier = d_refs.get('AcctSvcrRef', '')

260 d.end_to_end_identifier = d_refs.get('EndToEndId', '')

261

262 d_parties = dtl.get('RltdPties', {})

263 d_dbt = d_parties.get('Dbtr', {})

264 d.debtor_name = d_dbt.get('Nm', '')

265 d_udbt = d_parties.get('UltmtDbtr', {})

266 d.ultimate_debtor_name = d_udbt.get('Nm', '')

267 d_cdtr = d_parties.get('Cdtr', {})

268 d.creditor_name = d_cdtr.get('Nm', '')

269 d_cdtr_acct = d_parties.get('CdtrAcct', {})

270 d.creditor_account = d_cdtr_acct.get('Id', {}).get('IBAN', '')

271

272 d_rmt = dtl.get('RmtInf', {})

273 d.unstructured_remittance_info = d_rmt.get('Ustrd', '')

274

275 d_rltd_dts = dtl.get('RltdDts', {})

276 d.paid_date = camt053_get_dt(d_rltd_dts, 'AccptncDtTm') if 'AccptncDtTm' in d_rltd_dts else None

277

278 d.full_clean()

279 d.save()

280

281 st = StatementRecordRemittanceInfo(detail=d)

282 for strd in d_rmt.get('Strd', []):

283 additional_info = strd.get('AddtlRmtInf', '')

284 amount, currency_code = camt053_get_currency(strd.get('RfrdDocAmt', {}), 'RmtdAmt', required=False)

285 reference = strd.get('CdtrRefInf', {}).get('Ref', '')

286

287 # check if new remittance info record is needed

288 if additional_info and st.additional_info or amount and st.amount or \

289 reference and st.reference: # pylint: disable=too-many-boolean-expressions

290 st = StatementRecordRemittanceInfo(detail=d)

291

292 if additional_info:

293 st.additional_info = additional_info

294 if amount:

295 st.amount, st.currency_code = amount, currency_code

296 if reference:

297 st.reference = reference

298

299 st.full_clean()

300 st.save()

301

302 dtl_ix += 1

303

304 # fill record name from details

305 assert rec.type

306 if not rec.name:

307 if rec.type.code == e_withdraw.code:

308 rec.name = camt053_get_unified_str(rec.detail_set.all(), 'creditor_name')

309 elif rec.type.code == e_deposit.code:

310 rec.name = camt053_get_unified_str(rec.detail_set.all(), 'debtor_name')

311 if not rec.recipient_account_number:

312 rec.recipient_account_number = camt053_get_unified_str(rec.detail_set.all(), 'creditor_account')

313 if not rec.remittance_info:

314 rec.remittance_info = camt053_get_unified_str(StatementRecordRemittanceInfo.objects.all().filter(detail__record=rec), 'reference')

315 if not rec.paid_date:

316 paid_date = camt053_get_unified_val(rec.detail_set.all(), 'paid_date', default=None)

317 if paid_date:

318 assert isinstance(paid_date, datetime)

319 rec.paid_date = paid_date.date()

320

321 rec.full_clean()

322 rec.save()

323

324 return stm