Coverage for jbank/models.py: 79%

1 import base64

2 import logging

3 import os

4 import re

5 import subprocess

6 import tempfile

7 from datetime import datetime, time, date

8 from decimal import Decimal

9 from os.path import basename, join

10 from pathlib import Path

11 from typing import List, Optional

12 import pytz

13 from django.conf import settings

14 from django.core.exceptions import ValidationError

15 from django.db import models

16 from django.db.models import Q

17 from django.template.loader import get_template

18 from django.utils.timezone import now

19 from django.utils.translation import gettext_lazy as _

20 from jacc.helpers import sum_queryset

21 from jacc.models import AccountEntry, AccountEntrySourceFile, Account, AccountEntryManager

22 from jbank.x509_helpers import get_x509_cert_from_file

23 from jutil.modelfields import SafeCharField, SafeTextField

24 from jutil.format import format_xml, get_media_full_path, choices_label

25 from jutil.validators import iban_validator, iban_bic, iso_payment_reference_validator, fi_payment_reference_validator

26

27

28 logger = logging.getLogger(__name__)

29

30

31 JBANK_BIN_PATH = Path(__file__).absolute().parent.joinpath("bin")

32

33 RECORD_ENTRY_TYPE = (

34 ("1", _("Deposit")),

35 ("2", _("Withdrawal")),

36 ("3", _("Deposit Correction")),

37 ("4", _("Withdrawal Correction")),

38)

39

40 RECORD_CODES = (

41 ("700", _("Money Transfer (In/Out)")),

42 ("701", _("Recurring Payment (In/Out)")),

43 ("702", _("Bill Payment (Out)")),

44 ("703", _("Payment Terminal Deposit (In)")),

45 ("704", _("Bank Draft (In/Out)")),

46 ("705", _("Reference Payments (In)")),

47 ("706", _("Payment Service (Out)")),

48 ("710", _("Deposit (In)")),

49 ("720", _("Withdrawal (Out)")),

50 ("721", _("Card Payment (Out)")),

51 ("722", _("Check (Out)")),

52 ("730", _("Bank Fees (Out)")),

53 ("740", _("Interests Charged (Out)")),

54 ("750", _("Interests Credited (In)")),

55 ("760", _("Loan (Out)")),

56 ("761", _("Loan Payment (Out)")),

57 ("770", _("Foreign Transfer (In/Out)")),

58 ("780", _("Zero Balancing (In/Out)")),

59 ("781", _("Sweeping (In/Out)")),

60 ("782", _("Topping (In/Out)")),

61)

62

63 RECORD_DOMAIN = (

64 ("PMNT", _("Money Transfer (In/Out)")),

65 ("LDAS", _("Loan Payment (Out)")),

66 ("CAMT", _("Cash Management")),

67 ("ACMT", _("Account Management")),

68 ("XTND", _("Entended Domain")),

69 ("SECU", _("Securities")),

70 ("FORX", _("Foreign Exchange")),

71 ("XTND", _("Entended Domain")),

72 ("NTAV", _("Not Available")),

73)

74

75 RECEIPT_CODE = (

76 ("", ""),

77 ("0", "(0)"),

78 ("E", _("Separate")),

79 ("P", _("Separate/Paper")),

80)

81

82 CURRENCY_IDENTIFIERS = (("1", "EUR"),)

83

84 NAME_SOURCES = (

85 ("", _("Not Set")),

86 ("A", _("From Customer")),

87 ("K", _("From Bank Clerk")),

88 ("J", _("From Bank System")),

89)

90

91 CORRECTION_IDENTIFIER = (

92 ("0", _("Regular Entry")),

93 ("1", _("Correction Entry")),

94)

95

96 DELIVERY_METHOD_UNKNOWN = ""

97 DELIVERY_FROM_CUSTOMER = "A"

98 DELIVERY_FROM_BANK_CLERK = "K"

99 DELIVERY_FROM_BANK_SYSTEM = "J"

100

101 DELIVERY_METHOD = (

102 (DELIVERY_METHOD_UNKNOWN, ""),

103 (DELIVERY_FROM_CUSTOMER, _("From Customer")),

104 (DELIVERY_FROM_BANK_CLERK, _("From Bank Clerk")),

105 (DELIVERY_FROM_BANK_SYSTEM, _("From Bank System")),

106)

107

108 PAYOUT_ON_HOLD = "H"

109 PAYOUT_WAITING_PROCESSING = "W"

110 PAYOUT_WAITING_UPLOAD = "U"

111 PAYOUT_UPLOADED = "D"

112 PAYOUT_PAID = "P"

113 PAYOUT_CANCELED = "C"

114 PAYOUT_ERROR = "E"

115

116 PAYOUT_STATE = (

117 (PAYOUT_ON_HOLD, _("on hold")),

118 (PAYOUT_WAITING_PROCESSING, _("waiting processing")),

119 (PAYOUT_WAITING_UPLOAD, _("waiting upload")),

120 (PAYOUT_UPLOADED, _("uploaded")),

121 (PAYOUT_PAID, _("paid")),

122 (PAYOUT_CANCELED, _("canceled")),

123 (PAYOUT_ERROR, _("error")),

124)

125

126

127 class Statement(AccountEntrySourceFile):

128 file = models.ForeignKey("StatementFile", blank=True, default=None, null=True, on_delete=models.CASCADE)

129 account = models.ForeignKey(Account, related_name="+", on_delete=models.PROTECT)

130 account_number = SafeCharField(_("account number"), max_length=32, db_index=True)

131 statement_identifier = SafeCharField(_("statement identifier"), max_length=48, db_index=True, blank=True, default="")

132 statement_number = models.SmallIntegerField(_("statement number"), db_index=True)

133 begin_date = models.DateField(_("begin date"), db_index=True)

134 end_date = models.DateField(_("end date"), db_index=True)

135 record_date = models.DateTimeField(_("record date"), db_index=True)

136 customer_identifier = SafeCharField(_("customer identifier"), max_length=64, blank=True, default="")

137 begin_balance_date = models.DateField(_("begin balance date"), null=True, blank=True, default=None)

138 begin_balance = models.DecimalField(_("begin balance"), max_digits=10, decimal_places=2)

139 record_count = models.IntegerField(_("record count"), null=True, default=None)

140 currency_code = SafeCharField(_("currency code"), max_length=3)

141 account_name = SafeCharField(_("account name"), max_length=32, blank=True, default="")

142 account_limit = models.DecimalField(_("account limit"), max_digits=10, decimal_places=2, blank=True, default=None, null=True)

143 owner_name = SafeCharField(_("owner name"), max_length=64)

144 contact_info_1 = SafeCharField(_("contact info (1)"), max_length=64, blank=True, default="")

145 contact_info_2 = SafeCharField(_("contact info (2)"), max_length=64, blank=True, default="")

146 bank_specific_info_1 = SafeCharField(_("bank specific info (1)"), max_length=1024, blank=True, default="")

147 iban = SafeCharField(_("IBAN"), max_length=32, db_index=True)

148 bic = SafeCharField(_("BIC"), max_length=11, db_index=True)

149

150 class Meta:

151 verbose_name = _("statement")

152 verbose_name_plural = _("statements")

153

154

155 class PaymentRecordManager(AccountEntryManager):

156 def filter_matched(self):

157 return self.exclude(child_set=None)

158

159 def filter_unmatched(self):

160 return self.filter(child_set=None)

161

162

163 class StatementRecord(AccountEntry):

164 objects: models.Manager = PaymentRecordManager() # type: ignore

165 statement = models.ForeignKey(Statement, verbose_name=_("statement"), related_name="record_set", on_delete=models.CASCADE)

166 line_number = models.SmallIntegerField(_("line number"), default=None, null=True, blank=True)

167 record_number = models.IntegerField(_("record number"), default=None, null=True, blank=True)

168 archive_identifier = SafeCharField(_("archive identifier"), max_length=64, blank=True, default="", db_index=True)

169 record_date = models.DateField(_("record date"), db_index=True)

170 value_date = models.DateField(_("value date"), db_index=True, blank=True, null=True, default=None)

171 paid_date = models.DateField(_("paid date"), db_index=True, blank=True, null=True, default=None)

172 entry_type = SafeCharField(_("entry type"), max_length=1, choices=RECORD_ENTRY_TYPE, db_index=True)

173 record_code = SafeCharField(_("record type"), max_length=4, choices=RECORD_CODES, db_index=True, blank=True)

174 record_domain = SafeCharField(_("record domain"), max_length=4, choices=RECORD_DOMAIN, db_index=True, blank=True)

175 family_code = SafeCharField(_("family code"), max_length=4, db_index=True, blank=True, default="")

176 sub_family_code = SafeCharField(_("sub family code"), max_length=4, db_index=True, blank=True, default="")

177 record_description = SafeCharField(_("record description"), max_length=128, blank=True, default="")

178 receipt_code = SafeCharField(_("receipt code"), max_length=1, choices=RECEIPT_CODE, db_index=True, blank=True)

179 delivery_method = SafeCharField(_("delivery method"), max_length=1, db_index=True, choices=DELIVERY_METHOD, blank=True)

180 name = SafeCharField(_("name"), max_length=128, blank=True, db_index=True)

181 name_source = SafeCharField(_("name source"), max_length=1, blank=True, choices=NAME_SOURCES)

182 recipient_account_number = SafeCharField(_("recipient account number"), max_length=32, blank=True, db_index=True)

183 recipient_account_number_changed = SafeCharField(_("recipient account number changed"), max_length=1, blank=True)

184 remittance_info = SafeCharField(_("remittance info"), max_length=35, db_index=True, blank=True)

185 messages = SafeTextField(_("messages"), blank=True, default="")

186 client_messages = SafeTextField(_("client messages"), blank=True, default="")

187 bank_messages = SafeTextField(_("bank messages"), blank=True, default="")

188 manually_settled = models.BooleanField(_("manually settled"), db_index=True, default=False, blank=True)

189

190 class Meta:

191 verbose_name = _("statement record")

192 verbose_name_plural = _("statement records")

193

194 @property

195 def is_settled(self) -> bool:

196 """

197 True if entry is either manually settled or has SUM(children)==amount.

198 """

199 return self.manually_settled or sum_queryset(self.child_set) == self.amount # type: ignore

200

201 def clean(self):

202 self.source_file = self.statement

203 self.timestamp = pytz.utc.localize(datetime.combine(self.record_date, time(0, 0)))

204 if self.name:

205 self.description = "{name}: {record_description}".format(record_description=self.record_description, name=self.name)

206 else:

207 self.description = "{record_description}".format(record_description=self.record_description)

208

209

210 class CurrencyExchangeSource(models.Model):

211 name = SafeCharField(_("name"), max_length=64)

212 created = models.DateTimeField(_("created"), default=now, db_index=True, blank=True, editable=False)

213

214 class Meta:

215 verbose_name = _("currency exchange source")

216 verbose_name_plural = _("currency exchange sources")

217

218 def __str__(self):

219 return str(self.name)

220

221

222 class CurrencyExchange(models.Model):

223 record_date = models.DateField(_("record date"), db_index=True)

224 source_currency = SafeCharField(_("source currency"), max_length=3, blank=True)

225 target_currency = SafeCharField(_("target currency"), max_length=3, blank=True)

226 unit_currency = SafeCharField(_("unit currency"), max_length=3, blank=True)

227 exchange_rate = models.DecimalField(_("exchange rate"), decimal_places=6, max_digits=12, null=True, default=None, blank=True)

228 source = models.ForeignKey(

229 CurrencyExchangeSource,

230 verbose_name=_("currency exchange source"),

231 blank=True,

232 null=True,

233 default=None,

234 on_delete=models.PROTECT,

235 ) # noqa

236

237 class Meta:

238 verbose_name = _("currency exchange")

239 verbose_name_plural = _("currency exchanges")

240

241 def __str__(self):

242 return "{src} = {rate} {tgt}".format(src=self.source_currency, tgt=self.target_currency, rate=self.exchange_rate)

243

244

245 class StatementRecordDetail(models.Model):

246 record = models.ForeignKey(StatementRecord, verbose_name=_("record"), related_name="detail_set", on_delete=models.CASCADE)

247 batch_identifier = SafeCharField(_("batch message id"), max_length=64, db_index=True, blank=True, default="")

248 amount = models.DecimalField(verbose_name=_("amount"), max_digits=10, decimal_places=2, blank=True, default=None, null=True, db_index=True)

249 currency_code = SafeCharField(_("currency code"), max_length=3)

250 instructed_amount = models.DecimalField(

251 verbose_name=_("instructed amount"),

252 max_digits=10,

253 decimal_places=2,

254 blank=True,

255 default=None,

256 null=True,

257 db_index=True,

258 )

259 exchange = models.ForeignKey(

260 CurrencyExchange,

261 verbose_name=_("currency exchange"),

262 related_name="recorddetail_set",

263 on_delete=models.PROTECT,

264 null=True,

265 default=None,

266 blank=True,

267 )

268 archive_identifier = SafeCharField(_("archive identifier"), max_length=64, blank=True)

269 end_to_end_identifier = SafeCharField(_("end-to-end identifier"), max_length=64, blank=True)

270 creditor_name = SafeCharField(_("creditor name"), max_length=128, blank=True)

271 creditor_account = SafeCharField(_("creditor account"), max_length=35, blank=True)

272 creditor_account_scheme = SafeCharField(_("creditor account scheme"), max_length=8, blank=True)

273 debtor_name = SafeCharField(_("debtor name"), max_length=128, blank=True)

274 ultimate_debtor_name = SafeCharField(_("ultimate debtor name"), max_length=128, blank=True)

275 unstructured_remittance_info = SafeCharField(_("unstructured remittance info"), max_length=2048, blank=True)

276 paid_date = models.DateTimeField(_("paid date"), db_index=True, blank=True, null=True, default=None)

277

278 class Meta:

279 verbose_name = _("statement record details")

280 verbose_name_plural = _("statement record details")

281

282

283 class StatementRecordRemittanceInfo(models.Model):

284 detail = models.ForeignKey(StatementRecordDetail, related_name="remittanceinfo_set", on_delete=models.CASCADE)

285 additional_info = SafeCharField(_("additional remittance info"), max_length=256, blank=True, db_index=True)

286 amount = models.DecimalField(_("amount"), decimal_places=2, max_digits=10, null=True, default=None, blank=True)

287 currency_code = SafeCharField(_("currency code"), max_length=3, blank=True)

288 reference = SafeCharField(_("reference"), max_length=35, blank=True, db_index=True)

289

290 def __str__(self):

291 return "{} {} ref {} ({})".format(self.amount if self.amount is not None else "", self.currency_code, self.reference, self.additional_info)

292

293 class Meta:

294 verbose_name = _("statement record remittance info")

295 verbose_name_plural = _("statement record remittance info")

296

297

298 class StatementRecordSepaInfo(models.Model):

299 record = models.OneToOneField(StatementRecord, verbose_name=_("record"), related_name="sepa_info", on_delete=models.CASCADE)

300 reference = SafeCharField(_("reference"), max_length=35, blank=True)

301 iban_account_number = SafeCharField(_("IBAN"), max_length=35, blank=True)

302 bic_code = SafeCharField(_("BIC"), max_length=35, blank=True)

303 recipient_name_detail = SafeCharField(_("recipient name detail"), max_length=70, blank=True)

304 payer_name_detail = SafeCharField(_("payer name detail"), max_length=70, blank=True)

305 identifier = SafeCharField(_("identifier"), max_length=35, blank=True)

306 archive_identifier = SafeCharField(_("archive identifier"), max_length=64, blank=True)

307

308 class Meta:

309 verbose_name = _("SEPA")

310 verbose_name_plural = _("SEPA")

311

312 def __str__(self):

313 return "[{}]".format(self.id)

314

315

316 class ReferencePaymentBatchManager(models.Manager):

317 def latest_record_date(self) -> Optional[datetime]:

318 """

319 :return: datetime of latest record available or None

320 """

321 obj = self.order_by("-record_date").first()

322 if not obj:

323 return None

324 return obj.record_date

325

326

327 class ReferencePaymentBatch(AccountEntrySourceFile):

328 objects = ReferencePaymentBatchManager()

329 file = models.ForeignKey("ReferencePaymentBatchFile", blank=True, default=None, null=True, on_delete=models.CASCADE)

330 record_date = models.DateTimeField(_("record date"), db_index=True)

331 identifier = SafeCharField(_("institution"), max_length=32, blank=True)

332 institution_identifier = SafeCharField(_("institution"), max_length=2, blank=True, default="")

333 service_identifier = SafeCharField(_("service"), max_length=9, blank=True, default="")

334 currency_identifier = SafeCharField(_("currency"), max_length=3, blank=True, default="EUR")

335 cached_total_amount = models.DecimalField(_("total amount"), max_digits=10, decimal_places=2, null=True, default=None, blank=True)

336

337 class Meta:

338 verbose_name = _("reference payment batch")

339 verbose_name_plural = _("reference payment batches")

340

341 def get_total_amount(self, force: bool = False) -> Decimal:

342 if self.cached_total_amount is None or force:

343 self.cached_total_amount = sum_queryset(ReferencePaymentRecord.objects.filter(batch=self))

344 self.save(update_fields=["cached_total_amount"])

345 return self.cached_total_amount

346

347 @property

348 def total_amount(self) -> Decimal:

349 return self.get_total_amount()

350

351 total_amount.fget.short_description = _("total amount") # type: ignore

352

353

354 class ReferencePaymentRecord(AccountEntry):

355 """

356 Reference payment record. See jacc.Invoice for date/time variable naming conventions.

357 """

358

359 objects = PaymentRecordManager() # type: ignore

360 batch = models.ForeignKey(ReferencePaymentBatch, verbose_name=_("batch"), related_name="record_set", on_delete=models.CASCADE)

361 line_number = models.SmallIntegerField(_("line number"), default=0, blank=True)

362 record_type = SafeCharField(_("record type"), max_length=4, blank=True, default="")

363 account_number = SafeCharField(_("account number"), max_length=32, db_index=True)

364 record_date = models.DateField(_("record date"), db_index=True)

365 paid_date = models.DateField(_("paid date"), db_index=True, blank=True, null=True, default=None)

366 value_date = models.DateField(_("value date"), db_index=True, blank=True, null=True, default=None)

367 archive_identifier = SafeCharField(_("archive identifier"), max_length=32, blank=True, default="", db_index=True)

368 remittance_info = SafeCharField(_("remittance info"), max_length=256, db_index=True)

369 payer_name = SafeCharField(_("payer name"), max_length=64, blank=True, default="", db_index=True)

370 currency_identifier = SafeCharField(_("currency identifier"), max_length=1, choices=CURRENCY_IDENTIFIERS, blank=True, default="")

371 name_source = SafeCharField(_("name source"), max_length=1, choices=NAME_SOURCES, blank=True, default="")

372 correction_identifier = SafeCharField(_("correction identifier"), max_length=1, choices=CORRECTION_IDENTIFIER, default="")

373 delivery_method = SafeCharField(_("delivery method"), max_length=1, db_index=True, choices=DELIVERY_METHOD, blank=True, default="")

374 receipt_code = SafeCharField(_("receipt code"), max_length=1, choices=RECEIPT_CODE, db_index=True, blank=True, default="")

375 manually_settled = models.BooleanField(_("manually settled"), db_index=True, default=False, blank=True)

376 instructed_amount = models.DecimalField(_("instructed amount"), blank=True, default=None, null=True, max_digits=10, decimal_places=2)

377 instructed_currency = SafeCharField(_("instructed currency"), blank=True, default="", max_length=3)

378 creditor_bank_bic = SafeCharField(_("creditor bank BIC"), max_length=16, blank=True, default="")

379 end_to_end_identifier = SafeCharField(_("end to end identifier"), max_length=128, blank=True, default="")

380

381 class Meta:

382 verbose_name = _("reference payment records")

383 verbose_name_plural = _("reference payment records")

384

385 @property

386 def is_settled(self) -> bool:

387 """

388 True if entry is either manually settled or has SUM(children)==amount.

389 """

390 return self.manually_settled or sum_queryset(self.child_set) == self.amount # type: ignore

391

392 @property

393 def remittance_info_short(self) -> str:

394 """

395 Remittance info without preceding zeroes.

396 :return: str

397 """

398 return re.sub(r"^0+", "", self.remittance_info)

399

400 def clean(self):

401 self.source_file = self.batch

402 self.timestamp = pytz.utc.localize(datetime.combine(self.paid_date or self.record_date, time(0, 0)))

403 self.description = "{amount} {remittance_info} {payer_name}".format(

404 amount=self.amount, remittance_info=self.remittance_info, payer_name=self.payer_name

405 )

406

407

408 class StatementFile(models.Model):

409 created = models.DateTimeField(_("created"), default=now, db_index=True, blank=True, editable=False)

410 file = models.FileField(verbose_name=_("file"), upload_to="uploads")

411 original_filename = SafeCharField(_("original filename"), blank=True, default="", max_length=256)

412 tag = SafeCharField(_("tag"), blank=True, max_length=64, default="", db_index=True)

413 errors = SafeTextField(_("errors"), max_length=4086, default="", blank=True)

414

415 class Meta:

416 verbose_name = _("account statement file")

417 verbose_name_plural = _("account statement files")

418

419 @property

420 def full_path(self):

421 return join(settings.MEDIA_ROOT, self.file.name) if self.file else ""

422

423 def __str__(self):

424 return basename(str(self.file.name)) if self.file else ""

425

426

427 class ReferencePaymentBatchFile(models.Model):

428 created = models.DateTimeField(_("created"), default=now, db_index=True, blank=True, editable=False)

429 file = models.FileField(verbose_name=_("file"), upload_to="uploads")

430 original_filename = SafeCharField(_("original filename"), blank=True, default="", max_length=256)

431 tag = SafeCharField(_("tag"), blank=True, max_length=64, default="", db_index=True)

432 timestamp = models.DateTimeField(_("timestamp"), default=None, null=True, db_index=True, blank=True, editable=False)

433 msg_id = models.CharField(_("message identifier"), max_length=32, default="", blank=True, db_index=True)

434 additional_info = models.CharField(_("additional information"), max_length=128, default="", blank=True)

435 errors = SafeTextField(_("errors"), max_length=4086, default="", blank=True)

436 cached_total_amount = models.DecimalField(_("total amount"), max_digits=10, decimal_places=2, null=True, default=None, blank=True)

437

438 class Meta:

439 verbose_name = _("reference payment batch file")

440 verbose_name_plural = _("reference payment batch files")

441

442 def clean(self):

443 if self.timestamp is None:

444 self.timestamp = self.created

445

446 def get_total_amount(self, force: bool = False) -> Decimal:

447 if self.cached_total_amount is None or force:

448 self.cached_total_amount = sum_queryset(ReferencePaymentRecord.objects.filter(batch__file=self))

449 self.save(update_fields=["cached_total_amount"])

450 return self.cached_total_amount

451

452 @property

453 def total_amount(self) -> Decimal:

454 return self.get_total_amount()

455

456 total_amount.fget.short_description = _("total amount") # type: ignore

457

458 @property

459 def full_path(self):

460 return join(settings.MEDIA_ROOT, self.file.name) if self.file else ""

461

462 def __str__(self):

463 return basename(str(self.file.name)) if self.file else ""

464

465

466 class PayoutParty(models.Model):

467 name = SafeCharField(_("name"), max_length=128, db_index=True)

468 account_number = SafeCharField(_("account number"), max_length=35, db_index=True, validators=[iban_validator])

469 bic = SafeCharField(_("BIC"), max_length=16, db_index=True, blank=True)

470 org_id = SafeCharField(_("organization id"), max_length=32, db_index=True, blank=True, default="")

471 address = SafeTextField(_("address"), blank=True, default="")

472 country_code = SafeCharField(_("country code"), max_length=2, default="FI", blank=True, db_index=True)

473 payouts_account = models.ForeignKey(Account, verbose_name=_("payouts account"), null=True, default=None, blank=True, on_delete=models.PROTECT)

474

475 class Meta:

476 verbose_name = _("payout party")

477 verbose_name_plural = _("payout parties")

478

479 def __str__(self):

480 return "{} ({})".format(self.name, self.account_number)

481

482 @property

483 def is_payout_party_used(self) -> bool:

484 """

485 True if payout party has been used in any payment.

486 """

487 if not hasattr(self, "id") or self.id is None:

488 return False

489 return Payout.objects.all().filter(Q(recipient=self) | Q(payer=self)).exists()

490

491 @property

492 def is_account_number_changed(self) -> bool:

493 """

494 True if account number has been changed compared to the one stored in DB.

495 """

496 if not hasattr(self, "id") or self.id is None:

497 return False

498 return PayoutParty.objects.all().filter(id=self.id).exclude(account_number=self.account_number).exists()

499

500 def clean(self):

501 if not self.bic:

502 self.bic = iban_bic(self.account_number)

503 if self.is_account_number_changed and self.is_payout_party_used:

504 raise ValidationError({"account_number": _("Account number changes of used payout parties is not allowed. Create a new payout party instead.")})

505

506 @property

507 def address_lines(self):

508 out = []

509 for line in self.address.split("\n"):

510 line = line.strip()

511 if line:

512 out.append(line)

513 return out

514

515

516 class Payout(AccountEntry):

517 connection = models.ForeignKey(

518 "WsEdiConnection",

519 verbose_name=_("WS-EDI connection"),

520 on_delete=models.SET_NULL,

521 null=True,

522 blank=True,

523 related_name="+",

524 )

525 payer = models.ForeignKey(PayoutParty, verbose_name=_("payer"), related_name="+", on_delete=models.PROTECT)

526 recipient = models.ForeignKey(PayoutParty, verbose_name=_("recipient"), related_name="+", on_delete=models.PROTECT)

527 messages = SafeTextField(_("recipient messages"), blank=True, default="")

528 reference = SafeCharField(_("recipient reference"), blank=True, default="", max_length=32)

529 msg_id = SafeCharField(_("message id"), max_length=64, blank=True, db_index=True, editable=False)

530 file_name = SafeCharField(_("file name"), max_length=255, blank=True, db_index=True, editable=False)

531 full_path = SafeTextField(_("full path"), blank=True, editable=False)

532 file_reference = SafeCharField(_("file reference"), max_length=255, blank=True, db_index=True, editable=False)

533 due_date = models.DateField(_("due date"), db_index=True, blank=True, null=True, default=None)

534 paid_date = models.DateTimeField(_("paid date"), db_index=True, blank=True, null=True, default=None)

535 state = SafeCharField(_("state"), max_length=1, blank=True, default=PAYOUT_WAITING_PROCESSING, choices=PAYOUT_STATE, db_index=True)

536

537 class Meta:

538 verbose_name = _("payout")

539 verbose_name_plural = _("payouts")

540

541 def clean(self):

542 if self.parent and not self.amount:

543 self.amount = self.parent.amount

544

545 # prevent defining both reference and messages

546 if self.messages and self.reference or not self.messages and not self.reference:

547 raise ValidationError(_("payment.must.have.reference.or.messages"))

548

549 # validate reference if any

550 if self.reference:

551 if self.reference[:2] == "RF": # noqa

552 iso_payment_reference_validator(self.reference)

553 else:

554 fi_payment_reference_validator(self.reference)

555

556 # prevent canceling payouts which have been uploaded successfully

557 if self.state == PAYOUT_CANCELED:

558 if self.is_upload_done:

559 group_status = self.group_status

560 if group_status != "RJCT":

561 raise ValidationError(_("File already uploaded") + " ({})".format(group_status))

562

563 # save paid time if marking payout as paid manually

564 if self.state == PAYOUT_PAID and not self.paid_date:

565 self.paid_date = now()

566 status = self.payoutstatus_set.order_by("-created").first()

567 if status:

568 assert isinstance(status, PayoutStatus)

569 self.paid_date = status.created

570

571 # always require amount

572 if self.amount is None or self.amount <= Decimal("0.00"):

573 raise ValidationError({"amount": _("value > 0 required")})

574

575 def generate_msg_id(self, commit: bool = True):

576 msg_id_base = re.sub(r"[^\d]", "", now().isoformat())[:-4]

577 self.msg_id = msg_id_base + "P" + str(self.id)

578 if commit:

579 self.save(update_fields=["msg_id"])

580

581 @property

582 def state_name(self):

583 return choices_label(PAYOUT_STATE, self.state)

584

585 @property

586 def is_upload_done(self):

587 return PayoutStatus.objects.filter(payout=self, response_code="00").first() is not None

588

589 @property

590 def is_accepted(self):

591 return self.has_group_status("ACCP")

592

593 @property

594 def is_rejected(self):

595 return self.has_group_status("RJCT")

596

597 def has_group_status(self, group_status: str) -> bool:

598 return PayoutStatus.objects.filter(payout=self, group_status=group_status).exists()

599

600 @property

601 def group_status(self):

602 status = PayoutStatus.objects.filter(payout=self).order_by("-timestamp", "-id").first()

603 return status.group_status if status else ""

604

605 group_status.fget.short_description = _("payment.group.status") # type: ignore # pytype: disable=attribute-error

606

607

608 class PayoutStatusManager(models.Manager):

609 def is_file_processed(self, filename: str) -> bool:

610 return self.filter(file_name=basename(filename)).first() is not None

611

612

613 class PayoutStatus(models.Model):

614 objects = PayoutStatusManager()

615 payout = models.ForeignKey(

616 Payout,

617 verbose_name=_("payout"),

618 related_name="payoutstatus_set",

619 on_delete=models.PROTECT,

620 null=True,

621 default=None,

622 blank=True,

623 )

624 created = models.DateTimeField(_("created"), default=now, db_index=True, editable=False, blank=True)

625 timestamp = models.DateTimeField(_("timestamp"), default=now, db_index=True, editable=False, blank=True)

626 file_name = SafeCharField(_("file name"), max_length=128, blank=True, db_index=True, editable=False)

627 file_path = SafeCharField(_("file path"), max_length=255, blank=True, db_index=True, editable=False)

628 response_code = SafeCharField(_("response code"), max_length=4, blank=True, db_index=True)

629 response_text = SafeCharField(_("response text"), max_length=128, blank=True)

630 msg_id = SafeCharField(_("message id"), max_length=64, blank=True, db_index=True)

631 original_msg_id = SafeCharField(_("original message id"), blank=True, max_length=64, db_index=True)

632 group_status = SafeCharField(_("group status"), max_length=8, blank=True, db_index=True)

633 status_reason = SafeCharField(_("status reason"), max_length=255, blank=True)

634

635 class Meta:

636 verbose_name = _("payout status")

637 verbose_name_plural = _("payout statuses")

638

639 def __str__(self):

640 return str(self.group_status)

641

642 @property

643 def full_path(self) -> str:

644 return get_media_full_path(self.file_path) if self.file_path else ""

645

646 @property

647 def is_accepted(self):

648 return self.group_status == "ACCP"

649

650 @property

651 def is_rejected(self):

652 return self.group_status == "RJCT"

653

654

655 class Refund(Payout):

656 class Meta:

657 verbose_name = _("incoming.payment.refund")

658 verbose_name_plural = _("incoming.payment.refunds")

659

660 attachment = models.FileField(verbose_name=_("attachment"), blank=True, upload_to="uploads")

661

662

663 class WsEdiSoapCall(models.Model):

664 connection = models.ForeignKey("WsEdiConnection", verbose_name=_("WS-EDI connection"), on_delete=models.CASCADE)

665 command = SafeCharField(_("command"), max_length=64, blank=True, db_index=True)

666 created = models.DateTimeField(_("created"), default=now, db_index=True, editable=False, blank=True)

667 executed = models.DateTimeField(_("executed"), default=None, null=True, db_index=True, editable=False, blank=True)

668 error = SafeTextField(_("error"), blank=True)

669

670 class Meta:

671 verbose_name = _("WS-EDI SOAP call")

672 verbose_name_plural = _("WS-EDI SOAP calls")

673

674 def __str__(self):

675 return "WsEdiSoapCall({})".format(self.id)

676

677 @property

678 def timestamp(self) -> datetime:

679 return self.created.astimezone(pytz.timezone("Europe/Helsinki"))

680

681 @property

682 def timestamp_digits(self) -> str:

683 v = re.sub(r"[^\d]", "", self.created.isoformat())

684 return v[:17]

685

686 @property

687 def request_identifier(self) -> str:

688 return str(self.id)

689

690 @property

691 def command_camelcase(self) -> str:

692 return self.command[0:1].lower() + self.command[1:] # noqa

693

694 def debug_get_filename(self, file_type: str) -> str:

695 return "{:08}{}.xml".format(self.id, file_type)

696

697 @property

698 def debug_request_full_path(self) -> str:

699 return self.debug_get_file_path(self.debug_get_filename("q"))

700

701 @property

702 def debug_response_full_path(self) -> str:

703 return self.debug_get_file_path(self.debug_get_filename("s"))

704

705 @staticmethod

706 def debug_get_file_path(filename: str) -> str:

707 return os.path.join(settings.WSEDI_LOG_PATH, filename) if hasattr(settings, "WSEDI_LOG_PATH") and settings.WSEDI_LOG_PATH else ""

708

709

710 class WsEdiConnectionManager(models.Manager):

711 def get_by_receiver_identifier(self, receiver_identifier: str):

712 objs = list(self.filter(receiver_identifier=receiver_identifier))

713 if len(objs) != 1:

714 raise ValidationError(

715 _("WS-EDI connection cannot be found by receiver identifier {receiver_identifier} since there are {matches} matches").format(

716 receiver_identifier=receiver_identifier, matches=len(objs)

717 )

718 )

719 return objs[0]

720

721

722 class WsEdiConnection(models.Model):

723 objects = WsEdiConnectionManager()

724 name = SafeCharField(_("name"), max_length=64)

725 enabled = models.BooleanField(_("enabled"), blank=True, default=True)

726 sender_identifier = SafeCharField(_("sender identifier"), max_length=32)

727 receiver_identifier = SafeCharField(_("receiver identifier"), max_length=32)

728 target_identifier = SafeCharField(_("target identifier"), max_length=32)

729 environment = SafeCharField(_("environment"), max_length=32, default="PRODUCTION")

730 pin = SafeCharField("PIN", max_length=64, default="", blank=True)

731 pki_endpoint = models.URLField(_("PKI endpoint"), blank=True, default="")

732 bank_root_cert_file = models.FileField(verbose_name=_("bank root certificate file"), blank=True, upload_to="certs")

733 soap_endpoint = models.URLField(_("EDI endpoint"))

734 signing_cert_file = models.FileField(verbose_name=_("signing certificate file"), blank=True, upload_to="certs")

735 signing_key_file = models.FileField(verbose_name=_("signing key file"), blank=True, upload_to="certs")

736 encryption_cert_file = models.FileField(verbose_name=_("encryption certificate file"), blank=True, upload_to="certs")

737 encryption_key_file = models.FileField(verbose_name=_("encryption key file"), blank=True, upload_to="certs")

738 bank_encryption_cert_file = models.FileField(verbose_name=_("bank encryption cert file"), blank=True, upload_to="certs")

739 bank_signing_cert_file = models.FileField(verbose_name=_("bank signing cert file"), blank=True, upload_to="certs")

740 ca_cert_file = models.FileField(verbose_name=_("CA certificate file"), blank=True, upload_to="certs")

741 debug_commands = SafeTextField(_("debug commands"), blank=True, help_text=_("wsedi.connection.debug.commands.help.text"))

742 created = models.DateTimeField(_("created"), default=now, db_index=True, editable=False, blank=True)

743 _signing_cert = None

744

745 class Meta:

746 verbose_name = _("WS-EDI connection")

747 verbose_name_plural = _("WS-EDI connections")

748

749 def __str__(self):

750 return "{} / {}".format(self.name, self.receiver_identifier)

751

752 @property

753 def is_test(self) -> bool:

754 return str(self.environment).lower() in ["customertest", "test"]

755

756 @property

757 def signing_cert_full_path(self) -> str:

758 return get_media_full_path(self.signing_cert_file.file.name) if self.signing_cert_file else ""

759

760 @property

761 def signing_key_full_path(self) -> str:

762 return get_media_full_path(self.signing_key_file.file.name) if self.signing_key_file else ""

763

764 @property

765 def encryption_cert_full_path(self) -> str:

766 return get_media_full_path(self.encryption_cert_file.file.name) if self.encryption_cert_file else ""

767

768 @property

769 def encryption_key_full_path(self) -> str:

770 return get_media_full_path(self.encryption_key_file.file.name) if self.encryption_key_file else ""

771

772 @property

773 def bank_encryption_cert_full_path(self) -> str:

774 return get_media_full_path(self.bank_encryption_cert_file.file.name) if self.bank_encryption_cert_file else ""

775

776 @property

777 def bank_root_cert_full_path(self) -> str:

778 return get_media_full_path(self.bank_root_cert_file.file.name) if self.bank_root_cert_file else ""

779

780 @property

781 def ca_cert_full_path(self) -> str:

782 return get_media_full_path(self.ca_cert_file.file.name) if self.ca_cert_file else ""

783

784 @property

785 def signing_cert_with_public_key_full_path(self) -> str:

786 src_file = self.signing_cert_full_path

787 file = src_file[:-4] + "-with-pubkey.pem"

788 if not os.path.isfile(file):

789 cmd = [

790 settings.OPENSSL_PATH,

791 "x509",

792 "-pubkey",

793 "-in",

794 src_file,

795 ]

796 logger.info(" ".join(cmd))

797 out = subprocess.check_output(cmd)

798 with open(file, "wb") as fp:

799 fp.write(out)

800 return file

801

802 @property

803 def bank_encryption_cert_with_public_key_full_path(self) -> str:

804 src_file = self.bank_encryption_cert_full_path

805 file = src_file[:-4] + "-with-pubkey.pem"

806 if not os.path.isfile(file):

807 cmd = [

808 settings.OPENSSL_PATH,

809 "x509",

810 "-pubkey",

811 "-in",

812 src_file,

813 ]

814 # logger.info(' '.join(cmd))

815 out = subprocess.check_output(cmd)

816 with open(file, "wb") as fp:

817 fp.write(out)

818 return file

819

820 @property

821 def signing_cert(self):

822 if hasattr(self, "_signing_cert") and self._signing_cert:

823 return self._signing_cert

824 self._signing_cert = get_x509_cert_from_file(self.signing_cert_full_path)

825 return self._signing_cert

826

827 def get_pki_template(self, template_name: str, soap_call: WsEdiSoapCall, **kwargs) -> bytes:

828 return format_xml(

829 get_template(template_name).render(

830 {

831 "ws": soap_call.connection,

832 "soap_call": soap_call,

833 "command": soap_call.command,

834 "timestamp": now().astimezone(pytz.timezone("Europe/Helsinki")).isoformat(),

835 **kwargs,

836 }

837 )

838 ).encode()

839

840 def get_application_request(self, command: str, **kwargs) -> bytes:

841 return format_xml(

842 get_template("jbank/application_request_template.xml").render(

843 {

844 "ws": self,

845 "command": command,

846 "timestamp": now().astimezone(pytz.timezone("Europe/Helsinki")).isoformat(),

847 **kwargs,

848 }

849 )

850 ).encode()

851

852 @classmethod

853 def verify_signature(cls, content: bytes, signing_key_full_path: str):

854 with tempfile.NamedTemporaryFile() as fp:

855 fp.write(content)

856 fp.flush()

857 cmd = [settings.XMLSEC1_PATH, "--verify", "--pubkey-pem", signing_key_full_path, fp.name]

858 # logger.info(' '.join(cmd))

859 subprocess.check_output(cmd)

860

861 def sign_pki_request(self, content: bytes, signing_key_full_path: str, signing_cert_full_path: str) -> bytes:

862 return self._sign_request(content, signing_key_full_path, signing_cert_full_path)

863

864 def sign_application_request(self, content: bytes) -> bytes:

865 return self._sign_request(content, self.signing_key_full_path, self.signing_cert_full_path)

866

867 @classmethod

868 def _sign_request(cls, content: bytes, signing_key_full_path: str, signing_cert_full_path: str) -> bytes:

869 """

870 Sign a request.

871 See https://users.dcc.uchile.cl/~pcamacho/tutorial/web/xmlsec/xmlsec.html

872 :param content: XML application request

873 :param signing_key_full_path: Override signing key full path (if not use self.signing_key_full_path)

874 :param signing_cert_full_path: Override signing key full path (if not use self.signing_cert_full_path)

875 :return: str

876 """

877 with tempfile.NamedTemporaryFile() as fp:

878 fp.write(content)

879 fp.flush()

880 cmd = [

881 settings.XMLSEC1_PATH,

882 "--sign",

883 "--privkey-pem",

884 "{},{}".format(signing_key_full_path, signing_cert_full_path),

885 fp.name,

886 ]

887 # logger.info(' '.join(cmd))

888 out = subprocess.check_output(cmd)

889 cls.verify_signature(out, signing_key_full_path)

890 return out

891

892 def encrypt_pki_request(self, content: bytes) -> bytes:

893 return self._encrypt_request(content)

894

895 def encrypt_application_request(self, content: bytes) -> bytes:

896 return self._encrypt_request(content)

897

898 def _encrypt_request(self, content: bytes) -> bytes:

899 with tempfile.NamedTemporaryFile() as fp:

900 fp.write(content)

901 fp.flush()

902 cmd = [

903 self._xmlsec1_example_bin("encrypt3"),

904 fp.name,

905 self.bank_encryption_cert_with_public_key_full_path,

906 self.bank_encryption_cert_full_path,

907 ]

908 # logger.info(' '.join(cmd))

909 out = subprocess.check_output(cmd)

910 return out

911

912 def encode_application_request(self, content: bytes) -> bytes:

913 lines = content.split(b"\n")

914 if lines and lines[0].startswith(b"<?xml"):

915 lines = lines[1:]

916 content_without_xml_tag = b"\n".join(lines)

917 return base64.b64encode(content_without_xml_tag)

918

919 def decode_application_response(self, content: bytes) -> bytes:

920 return base64.b64decode(content)

921

922 def decrypt_application_response(self, content: bytes) -> bytes:

923 with tempfile.NamedTemporaryFile() as fp:

924 fp.write(content)

925 fp.flush()

926 cmd = [

927 self._xmlsec1_example_bin("decrypt3"),

928 fp.name,

929 self.encryption_key_full_path,

930 ]

931 # logger.info(' '.join(cmd))

932 out = subprocess.check_output(cmd)

933 return out

934

935 @property

936 def debug_command_list(self) -> List[str]:

937 return [x for x in re.sub(r"[^\w]+", " ", self.debug_commands).strip().split(" ") if x]

938

939 @staticmethod

940 def _xmlsec1_example_bin(file: str) -> str:

941 if hasattr(settings, "XMLSEC1_EXAMPLES_PATH") and settings.XMLSEC1_EXAMPLES_PATH:

942 xmlsec1_examples_path = settings.XMLSEC1_EXAMPLES_PATH

943 else:

944 xmlsec1_examples_path = os.path.join(str(os.getenv("HOME") or ""), "bin/xmlsec1-examples")

945 return str(os.path.join(xmlsec1_examples_path, file))

946

947

948 class EuriborRateManager(models.Manager):

949 def save_unique(self, record_date: date, name: str, rate: Decimal):

950 return self.get_or_create(record_date=record_date, name=name, defaults={"rate": rate})[0]

951

952

953 class EuriborRate(models.Model):

954 objects = EuriborRateManager()

955 record_date = models.DateField(_("record date"), db_index=True)

956 name = SafeCharField(_("interest rate name"), db_index=True, max_length=64)

957 rate = models.DecimalField(_("interest rate %"), max_digits=10, decimal_places=4, db_index=True)

958 created = models.DateTimeField(_("created"), default=now, db_index=True, blank=True, editable=False)

959

960 class Meta:

961 verbose_name = _("euribor rate")

962 verbose_name_plural = _("euribor rates")

963

964

965 class AccountBalance(models.Model):

966 account_number = models.CharField(_("account number"), max_length=32, db_index=True)

967 bic = models.CharField("BIC", max_length=16, db_index=True)

968 record_datetime = models.DateTimeField(_("record date"), db_index=True)

969 balance = models.DecimalField(_("balance"), max_digits=10, decimal_places=2)

970 available_balance = models.DecimalField(_("available balance"), max_digits=10, decimal_places=2)

971 credit_limit = models.DecimalField(_("credit limit"), max_digits=10, decimal_places=2, null=True, default=None, blank=True)

972 currency = models.CharField(_("currency"), max_length=3, default="EUR", db_index=True)

973 created = models.DateTimeField(_("created"), default=now, db_index=True, blank=True, editable=False)

974

975 class Meta:

976 verbose_name = _("account balance")

977 verbose_name_plural = _("account balances")