Coverage for jbank/svm.py: 90%

1 from os.path import basename

2 from typing import Union, Dict, List, Optional, Any

3

4 from django.conf import settings

5 from django.core.exceptions import ValidationError

6 from django.db import transaction

7 from django.utils.translation import gettext as _, gettext_lazy as _

8 from jacc.models import Account, EntryType

9 from pytz import timezone

10

11 from jbank.helpers import MESSAGE_STATEMENT_RECORD_FIELDS

12 from jbank.models import StatementFile, Statement, StatementRecord, StatementRecordSepaInfo, ReferencePaymentBatchFile, ReferencePaymentBatch, \

13 ReferencePaymentRecord

14 from jbank.parsers import parse_filename_suffix, parse_records, convert_date_fields, convert_decimal_fields

15

16 SVM_STATEMENT_SUFFIXES = ("SVM", "TXT", "KTL")

17

18 SVM_FILE_HEADER_DATES = (("record_date", "record_time"),)

19

20 SVM_FILE_HEADER_TYPES = ("0",)

21

22 SVM_FILE_HEADER = (

23 ("statement_type", "9(1)", "P"),

24 ("record_date", "9(6)", "P"),

25 ("record_time", "9(4)", "P"),

26 ("institution_identifier", "X(2)", "P"),

27 ("service_identifier", "X(9)", "P"),

28 ("currency_identifier", "X(1)", "P"),

29 ("pad01", "X(67)", "P"),

30)

31

32 SVM_FILE_RECORD_TYPES = ("3", "5")

33

34 SVM_FILE_RECORD_DECIMALS = ("amount",)

35

36 SVM_FILE_RECORD_DATES = (

37 "record_date",

38 "paid_date",

39)

40

41 SVM_FILE_RECORD = (

42 ("record_type", "9(1)", "P"), # 3=viitesiirto, 5=suoraveloitus

43 ("account_number", "9(14)", "P"),

44 ("record_date", "9(6)", "P"),

45 ("paid_date", "9(6)", "P"),

46 ("archive_identifier", "X(16)", "P"),

47 ("remittance_info", "X(20)", "P"),

48 ("payer_name", "X(12)", "P"),

49 ("currency_identifier", "X(1)", "P"), # 1=eur

50 ("name_source", "X", "V"),

51 ("amount", "9(10)", "P"),

52 ("correction_identifier", "X", "V"), # 0=normal, 1=correction

53 ("delivery_method", "X", "P"), # A=asiakkaalta, K=konttorista, J=pankin jarjestelmasta

54 ("receipt_code", "X", "P"),

55)

56

57 SVM_FILE_SUMMARY_TYPES = ("9",)

58

59 SVM_FILE_SUMMARY_DECIMALS = (

60 "record_amount",

61 "correction_amount",

62)

63

64 SVM_FILE_SUMMARY = (

65 ("record_type", "9(1)", "P"), # 9

66 ("record_count", "9(6)", "P"),

67 ("record_amount", "9(11)", "P"),

68 ("correction_count", "9(6)", "P"),

69 ("correction_amount", "9(11)", "P"),

70 ("pad01", "X(5)", "P"),

71)

72

73

74 def parse_svm_batches_from_file(filename: str) -> list:

75 if parse_filename_suffix(filename).upper() not in SVM_STATEMENT_SUFFIXES:

76 raise ValidationError(

77 _('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format(

78 filename=filename, suffixes=", ".join(SVM_STATEMENT_SUFFIXES), file_type="saapuvat viitemaksut"

79 )

80 )

81 with open(filename, "rt", encoding="ISO-8859-1") as fp:

82 return parse_svm_batches(fp.read(), filename=basename(filename)) # type: ignore

83

84

85 def parse_svm_batches(content: str, filename: str) -> list:

86 lines = content.split("\n")

87 nlines = len(lines)

88 line_number = 1

89 tz = timezone("Europe/Helsinki")

90 batches = []

91 header: Optional[Dict[str, Union[int, str]]] = None

92 records: List[Dict[str, Union[int, str]]] = []

93 summary: Optional[Dict[str, Union[int, str]]] = None

94

95 while line_number <= nlines:

96 line = lines[line_number - 1]

97 if line.strip() == "":

98 line_number += 1

99 continue

100 record_type = line[:1]

101

102 if record_type in SVM_FILE_HEADER_TYPES:

103 if header:

104 batches.append(combine_svm_batch(header, records, summary))

105 header, records, summary = None, [], None

106 header = parse_records(lines[line_number - 1], SVM_FILE_HEADER, line_number=line_number)

107 convert_date_fields(header, SVM_FILE_HEADER_DATES, tz)

108 line_number += 1

109 elif record_type in SVM_FILE_RECORD_TYPES:

110 record = parse_records(line, SVM_FILE_RECORD, line_number=line_number)

111 convert_date_fields(record, SVM_FILE_RECORD_DATES, tz)

112 convert_decimal_fields(record, SVM_FILE_RECORD_DECIMALS)

113 line_number += 1

114 records.append(record)

115 elif record_type in SVM_FILE_SUMMARY_TYPES:

116 summary = parse_records(line, SVM_FILE_SUMMARY, line_number=line_number)

117 convert_decimal_fields(summary, SVM_FILE_SUMMARY_DECIMALS)

118 line_number += 1

119 else:

120 raise ValidationError(_("Unknown record type on {}({}): {}").format(filename, line_number, record_type))

121

122 batches.append(combine_svm_batch(header, records, summary))

123 return batches

124

125

126 def combine_svm_batch(header: Optional[Dict[str, Any]], records: List[Dict[str, Union[int, str]]], summary: Optional[Dict[str, Any]]) -> Dict[str, Any]:

127 data = {"header": header, "records": records}

128 if summary is not None:

129 data["summary"] = summary

130 return data

131

132

133@transaction.atomic # noqa

134 def create_statement(statement_data: dict, name: str, file: StatementFile, **kw) -> Statement: # noqa

135 """

136 Creates Statement from statement data parsed by parse_tiliote_statements()

137 :param statement_data: See parse_tiliote_statements

138 :param name: File name of the account statement

139 :param file: Source statement file

140 :return: Statement

141 """

142 if "header" not in statement_data or not statement_data["header"]:

143 raise ValidationError("Invalid header field in statement data {}: {}".format(name, statement_data.get("header")))

144 header = statement_data["header"]

145

146 account_number = header["account_number"]

147 if not account_number:

148 raise ValidationError("{name}: ".format(name=name) + _("account.not.found").format(account_number=""))

149 accounts = list(Account.objects.filter(name=account_number))

150 if len(accounts) != 1:

151 raise ValidationError("{name}: ".format(name=name) + _("account.not.found").format(account_number=account_number))

152 account = accounts[0]

153 assert isinstance(account, Account)

154

155 if Statement.objects.filter(name=name, account=account).first():

156 raise ValidationError("Bank account {} statement {} of processed already".format(account_number, name))

157 stm = Statement(name=name, account=account, file=file)

158 for k in ASSIGNABLE_STATEMENT_HEADER_FIELDS:

159 if k in header:

160 setattr(stm, k, header[k])

161 # pprint(statement_data['header'])

162 for k, v in kw.items():

163 setattr(stm, k, v)

164 stm.full_clean()

165 stm.save()

166

167 if EntryType.objects.filter(code=settings.E_BANK_DEPOSIT).count() == 0:

168 raise ValidationError(_("entry.type.missing") + " ({}): {}".format("settings.E_BANK_DEPOSIT", settings.E_BANK_DEPOSIT))

169 if EntryType.objects.filter(code=settings.E_BANK_WITHDRAW).count() == 0:

170 raise ValidationError(_("entry.type.missing") + " ({}): {}".format("settings.E_BANK_WITHDRAW", settings.E_BANK_WITHDRAW))

171 entry_types = {

172 "1": EntryType.objects.get(code=settings.E_BANK_DEPOSIT),

173 "2": EntryType.objects.get(code=settings.E_BANK_WITHDRAW),

174 }

175

176 for rec_data in statement_data["records"]:

177 line_number = rec_data["line_number"]

178 e_type = entry_types.get(rec_data["entry_type"])

179 rec = StatementRecord(statement=stm, account=account, type=e_type, line_number=line_number)

180 for k in ASSIGNABLE_STATEMENT_RECORD_FIELDS:

181 if k in rec_data:

182 setattr(rec, k, rec_data[k])

183 for k in MESSAGE_STATEMENT_RECORD_FIELDS:

184 if k in rec_data:

185 setattr(rec, k, "\n".join(rec_data[k]))

186 rec.full_clean()

187 rec.save()

188

189 if "sepa" in rec_data:

190 sepa_info_data = rec_data["sepa"]

191 sepa_info = StatementRecordSepaInfo(record=rec)

192 for k in ASSIGNABLE_STATEMENT_RECORD_SEPA_INFO_FIELDS:

193 if k in sepa_info_data:

194 setattr(sepa_info, k, sepa_info_data[k])

195 # pprint(rec_data['sepa'])

196 sepa_info.full_clean()

197 sepa_info.save()

198

199 return stm

200

201

202@transaction.atomic

203 def create_reference_payment_batch(batch_data: dict, name: str, file: ReferencePaymentBatchFile, **kw) -> ReferencePaymentBatch:

204 """

205 Creates ReferencePaymentBatch from data parsed by parse_svm_batches()

206 :param batch_data: See parse_svm_batches

207 :param name: File name of the batch file

208 :return: ReferencePaymentBatch

209 """

210 if ReferencePaymentBatch.objects.exclude(file=file).filter(name=name).first():

211 raise ValidationError("Reference payment batch file {} already exists".format(name))

212

213 if "header" not in batch_data or not batch_data["header"]:

214 raise ValidationError("Invalid header field in reference payment batch data {}: {}".format(name, batch_data.get("header")))

215 header = batch_data["header"]

216

217 batch = ReferencePaymentBatch(name=name, file=file)

218 for k in ASSIGNABLE_REFERENCE_PAYMENT_BATCH_HEADER_FIELDS:

219 if k in header:

220 setattr(batch, k, header[k])

221 # pprint(statement_data['header'])

222 for k, v in kw.items():

223 setattr(batch, k, v)

224 batch.full_clean()

225 batch.save()

226 e_type = EntryType.objects.get(code=settings.E_BANK_REFERENCE_PAYMENT)

227

228 for rec_data in batch_data["records"]:

229 line_number = rec_data["line_number"]

230 account_number = rec_data["account_number"]

231 if not account_number:

232 raise ValidationError("{name}: ".format(name=name) + _("account.not.found").format(account_number=""))

233 accounts = list(Account.objects.filter(name=account_number))

234 if len(accounts) != 1:

235 raise ValidationError("{name}: ".format(name=name) + _("account.not.found").format(account_number=account_number))

236 account = accounts[0]

237 assert isinstance(account, Account)

238

239 rec = ReferencePaymentRecord(batch=batch, account=account, type=e_type, line_number=line_number)

240 for k in ASSIGNABLE_REFERENCE_PAYMENT_RECORD_FIELDS:

241 if k in rec_data:

242 setattr(rec, k, rec_data[k])

243 # pprint(rec_data)

244 rec.full_clean()

245 rec.save()

246

247 return batch

248

249

250 ASSIGNABLE_REFERENCE_PAYMENT_RECORD_FIELDS = (

251 "record_type",

252 "account_number",

253 "record_date",

254 "paid_date",

255 "archive_identifier",

256 "remittance_info",

257 "payer_name",

258 "currency_identifier",

259 "name_source",

260 "amount",

261 "correction_identifier",

262 "delivery_method",

263 "receipt_code",

264)

265 ASSIGNABLE_REFERENCE_PAYMENT_BATCH_HEADER_FIELDS = (

266 "record_date",

267 "institution_identifier",

268 "service_identifier",

269 "currency_identifier",

270)

271 ASSIGNABLE_STATEMENT_RECORD_SEPA_INFO_FIELDS = (

272 "reference",

273 "iban_account_number",

274 "bic_code",

275 "recipient_name_detail",

276 "payer_name_detail",

277 "identifier",

278 "archive_identifier",

279)

280 ASSIGNABLE_STATEMENT_RECORD_FIELDS = (

281 "record_date",

282 "value_date",

283 "paid_date",

284 "record_number",

285 "archive_identifier",

286 "entry_type",

287 "record_code",

288 "record_description",

289 "amount",

290 "receipt_code",

291 "delivery_method",

292 "name",

293 "name_source",

294 "recipient_account_number",

295 "recipient_account_number_changed",

296 "remittance_info",

297)

298 ASSIGNABLE_STATEMENT_HEADER_FIELDS = (

299 "account_number",

300 "statement_number",

301 "begin_date",

302 "end_date",

303 "record_date",

304 "customer_identifier",

305 "begin_balance_date",

306 "begin_balance",

307 "record_count",

308 "currency_code",

309 "account_name",

310 "account_limit",

311 "owner_name",

312 "contact_info_1",

313 "contact_info_2",

314 "bank_specific_info_1",

315 "iban",

316 "bic",

317)