Coverage for jbank/tito.py: 83%

1 from copy import copy

2 from os.path import basename

3 from typing import Dict, Any, List

4 from django.core.exceptions import ValidationError

5 from django.utils.translation import gettext as _

6 from pytz import timezone

7 from jbank.parsers import parse_filename_suffix, parse_records, convert_date_fields, convert_decimal_fields

8

9 TO_STATEMENT_SUFFIXES = ("TO", "TXT", "TITO")

10

11 TO_FILE_HEADER_TYPES = ("T00",)

12

13 TO_FILE_HEADER_DATES = (

14 "begin_date",

15 "end_date",

16 ("record_date", "record_time"),

17 "begin_balance_date",

18)

19

20 TO_FILE_HEADER_DECIMALS = (

21 ("begin_balance", "begin_balance_sign"),

22 "account_limit",

23)

24

25 TO_FILE_HEADER = (

26 ("statement_type", "X", "P"),

27 ("record_type", "XX", "P"),

28 ("record_length", "9(3)", "P"),

29 ("version", "X(3)", "P"),

30 ("account_number", "X(14)", "P"),

31 ("statement_number", "9(3)", "P"),

32 ("begin_date", "9(6)", "P"),

33 ("end_date", "9(6)", "P"),

34 ("record_date", "9(6)", "P"),

35 ("record_time", "9(4)", "P"),

36 ("customer_identifier", "X(17)", "P"),

37 ("begin_balance_date", "9(6)", "P"),

38 ("begin_balance_sign", "X", "P"),

39 ("begin_balance", "9(18)", "P"),

40 ("record_count", "9(6)", "P"),

41 ("currency_code", "X(3)", "P"),

42 ("account_name", "X(30)", "V"),

43 ("account_limit", "9(18)", "P"),

44 ("owner_name", "X(35)", "P"),

45 ("contact_info_1", "X(40)", "P"),

46 ("contact_info_2", "X(40)", "V"),

47 ("bank_specific_info_1", "X(30)", "V"),

48 ("iban_and_bic", "X(30)", "V"),

49)

50

51 TO_FILE_RECORD_TYPES = ("T10", "T80")

52

53 TO_FILE_RECORD_DATES = (

54 "record_date",

55 "value_date",

56 "paid_date",

57)

58

59 TO_FILE_RECORD_DECIMALS = (("amount", "amount_sign"),)

60

61 TO_FILE_RECORD = (

62 ("statement_type", "X", "P"),

63 ("record_type", "XX", "P"),

64 ("record_length", "9(3)", "P"),

65 ("record_number", "9(6)", "P"),

66 ("archive_identifier", "X(18)", "V"),

67 ("record_date", "9(6)", "P"),

68 ("value_date", "9(6)", "V"),

69 ("paid_date", "9(6)", "V"),

70 ("entry_type", "X", "P"), # 1 = pano, 2 = otto, 3 = panon korjaus, 4 = oton korjaus, 9 = hylätty tapahtuma

71 ("record_code", "X(3)", "P"),

72 ("record_description", "X(35)", "P"),

73 ("amount_sign", "X", "P"),

74 ("amount", "9(18)", "P"),

75 ("receipt_code", "X", "P"),

76 ("delivery_method", "X", "P"),

77 ("name", "X(35)", "V"),

78 ("name_source", "X", "V"),

79 ("recipient_account_number", "X(14)", "V"),

80 ("recipient_account_number_changed", "X", "V"),

81 ("remittance_info", "X(20)", "V"),

82 ("form_number", "X(8)", "V"),

83 ("level_identifier", "X", "P"),

84)

85

86 TO_FILE_RECORD_EXTRA_INFO_TYPES = ("T11", "T81")

87

88 TO_FILE_RECORD_EXTRA_INFO_HEADER = (

89 ("statement_type", "X", "P"),

90 ("record_type", "XX", "P"),

91 ("record_length", "9(3)", "P"),

92 ("extra_info_type", "9(2)", "P"),

93)

94

95 TO_FILE_RECORD_EXTRA_INFO_COUNTS = (("entry_count", "9(8)", "P"),)

96

97 TO_FILE_RECORD_EXTRA_INFO_INVOICE = (

98 ("customer_number", "X(10)", "P"),

99 ("pad01", "X", "P"),

100 ("invoice_number", "X(15)", "P"),

101 ("pad02", "X", "P"),

102 ("invoice_date", "X(6)", "P"),

103)

104

105 TO_FILE_RECORD_EXTRA_INFO_CARD = (

106 ("card_number", "X(19)", "P"),

107 ("pad01", "X", "P"),

108 ("merchant_reference", "X(14)", "P"),

109)

110

111 TO_FILE_RECORD_EXTRA_INFO_CORRECTION = (("original_archive_identifier", "X(18)", "P"),)

112

113 TO_FILE_RECORD_EXTRA_INFO_CURRENCY_DECIMALS = (("amount", "amount_sign"),)

114

115 TO_FILE_RECORD_EXTRA_INFO_CURRENCY = (

116 ("amount_sign", "X", "P"),

117 ("amount", "9(18)", "P"),

118 ("pad01", "X", "P"),

119 ("currency_code", "X(3)", "P"),

120 ("pad02", "X", "P"),

121 ("exchange_rate", "9(11)", "P"),

122 ("rate_reference", "X(6)", "V"),

123)

124

125 TO_FILE_RECORD_EXTRA_INFO_REASON = (

126 ("reason_code", "9(3)", "P"),

127 ("pad01", "X", "P"),

128 ("reason_description", "X(31)", "P"),

129)

130

131 TO_FILE_RECORD_EXTRA_INFO_NAME_DETAIL = (("name_detail", "X(35)", "P"),)

132

133 TO_FILE_RECORD_EXTRA_INFO_SEPA = (

134 ("reference", "X(35)", "V"),

135 ("iban_account_number", "X(35)", "V"),

136 ("bic_code", "X(35)", "V"),

137 ("recipient_name_detail", "X(70)", "V"),

138 ("payer_name_detail", "X(70)", "V"),

139 ("identifier", "X(35)", "V"),

140 ("archive_identifier", "X(35)", "V"),

141)

142

143 TO_BALANCE_RECORD_DATES = ("record_date",)

144

145 TO_BALANCE_RECORD_DECIMALS = (

146 ("end_balance", "end_balance_sign"),

147 ("available_balance", "available_balance_sign"),

148)

149

150 TO_BALANCE_RECORD = (

151 ("statement_type", "X", "P"),

152 ("record_type", "XX", "P"),

153 ("record_length", "9(3)", "P"),

154 ("record_date", "9(6)", "P"),

155 ("end_balance_sign", "X", "P"),

156 ("end_balance", "9(18)", "P"),

157 ("available_balance_sign", "X", "P"),

158 ("available_balance", "9(18)", "P"),

159)

160

161 TO_CUMULATIVE_RECORD_DATES = ("period_date",)

162

163 TO_CUMULATIVE_RECORD_DECIMALS = (

164 ("deposits_amount", "deposits_sign"),

165 ("withdrawals_amount", "withdrawals_sign"),

166)

167

168 TO_CUMULATIVE_RECORD = (

169 ("statement_type", "X", "P"),

170 ("record_type", "XX", "P"),

171 ("record_length", "9(3)", "P"),

172 ("period_identifier", "X", "P"), # 1=day, 2=term, 3=month, 4=year

173 ("period_date", "9(6)", "P"),

174 ("deposits_count", "9(8)", "P"),

175 ("deposits_sign", "X", "P"),

176 ("deposits_amount", "9(18)", "P"),

177 ("withdrawals_count", "9(8)", "P"),

178 ("withdrawals_sign", "X", "P"),

179 ("withdrawals_amount", "9(18)", "P"),

180)

181

182 TO_SPECIAL_RECORD = (

183 ("statement_type", "X", "P"),

184 ("record_type", "XX", "P"),

185 ("record_length", "9(3)", "P"),

186 ("bank_group_identifier", "X(3)", "P"),

187)

188

189

190 def parse_tiliote_statements_from_file(filename: str) -> list:

191 if parse_filename_suffix(filename).upper() not in TO_STATEMENT_SUFFIXES:

192 raise ValidationError(

193 _('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"').format(

194 filename=filename, suffixes=", ".join(TO_STATEMENT_SUFFIXES), file_type="tiliote"

195 )

196 )

197 with open(filename, "rt", encoding="ISO-8859-1") as fp:

198 return parse_tiliote_statements(fp.read(), filename=basename(filename)) # type: ignore

199

200

201 def parse_tiliote_statements(content: str, filename: str) -> List[dict]: # pylint: disable=too-many-locals

202 lines = content.split("\n")

203 nlines = len(lines)

204

205 line_number = 1

206 tz = timezone("Europe/Helsinki")

207

208 header = None

209 records = []

210 balance = None

211 cumulative = None

212 cumulative_adjustment = None

213 special_records = []

214 statements = []

215

216 while line_number <= nlines:

217 line = lines[line_number - 1]

218 if line.strip() == "":

219 line_number += 1

220 continue

221 # print('parsing line {}: {}'.format(line_number, line))

222 record_type = line[:3]

223

224 if record_type in TO_FILE_HEADER_TYPES:

225 if header:

226 statements.append(combine_statement(header, records, balance, cumulative, cumulative_adjustment, special_records))

227 header, records, balance, cumulative, cumulative_adjustment, special_records = (

228 None,

229 [],

230 None,

231 None,

232 None,

233 [],

234 )

235

236 header = parse_records(lines[line_number - 1], TO_FILE_HEADER, line_number=line_number)

237 convert_date_fields(header, TO_FILE_HEADER_DATES, tz)

238 convert_decimal_fields(header, TO_FILE_HEADER_DECIMALS)

239 iban_and_bic = str(header.get("iban_and_bic", "")).split(" ")

240 if len(iban_and_bic) == 2:

241 header["iban"], header["bic"] = iban_and_bic

242 line_number += 1

243

244 elif record_type in TO_FILE_RECORD_TYPES:

245 record = parse_records(line, TO_FILE_RECORD, line_number=line_number)

246 convert_date_fields(record, TO_FILE_RECORD_DATES, tz)

247 convert_decimal_fields(record, TO_FILE_RECORD_DECIMALS)

248

249 line_number += 1

250 # check for record extra info

251 if line_number <= nlines:

252 line = lines[line_number - 1]

253 while line[:3] in TO_FILE_RECORD_EXTRA_INFO_TYPES:

254 parse_record_extra_info(record, line, line_number=line_number)

255 line_number += 1

256 line = lines[line_number - 1]

257

258 records.append(record)

259 elif record_type == "T40":

260 balance = parse_records(line, TO_BALANCE_RECORD, line_number=line_number)

261 convert_date_fields(balance, TO_BALANCE_RECORD_DATES, tz)

262 convert_decimal_fields(balance, TO_BALANCE_RECORD_DECIMALS)

263 line_number += 1

264 elif record_type == "T50":

265 cumulative = parse_records(line, TO_CUMULATIVE_RECORD, line_number=line_number)

266 convert_date_fields(cumulative, TO_CUMULATIVE_RECORD_DATES, tz)

267 convert_decimal_fields(cumulative, TO_CUMULATIVE_RECORD_DECIMALS)

268 line_number += 1

269 elif record_type == "T51":

270 cumulative_adjustment = parse_records(line, TO_CUMULATIVE_RECORD, line_number=line_number)

271 convert_date_fields(cumulative_adjustment, TO_CUMULATIVE_RECORD_DATES, tz)

272 convert_decimal_fields(cumulative_adjustment, TO_CUMULATIVE_RECORD_DECIMALS)

273 line_number += 1

274 elif record_type in ("T60", "T70"):

275 special_records.append(parse_records(line, TO_SPECIAL_RECORD, line_number=line_number, check_record_length=False))

276 line_number += 1

277 else:

278 raise ValidationError(_("Unknown record type on {}({}): {}").format(filename, line_number, record_type))

279

280 statements.append(combine_statement(header, records, balance, cumulative, cumulative_adjustment, special_records))

281 return statements

282

283

284 def combine_statement(header, records, balance, cumulative, cumulative_adjustment, special_records) -> Dict[str, Any]: # pylint: disable=too-many-arguments

285 data = {

286 "header": header,

287 "records": records,

288 }

289 if balance is not None:

290 data["balance"] = balance

291 if cumulative is not None:

292 data["cumulative"] = cumulative

293 if cumulative_adjustment is not None:

294 data["cumulative_adjustment"] = cumulative_adjustment

295 if special_records:

296 data["special_records"] = special_records

297 return data

298

299

300 def parse_record_extra_info(record: Dict[str, Any], line: str, line_number: int):

301 if line[:3] not in TO_FILE_RECORD_EXTRA_INFO_TYPES:

302 raise ValidationError("SVM record extra info validation error on line {}".format(line_number))

303

304 header = parse_records(line, TO_FILE_RECORD_EXTRA_INFO_HEADER, line_number, check_record_length=False)

305 extra_info_type = header["extra_info_type"]

306 # print(line)

307 extra_data = copy(header["extra_data"])

308 assert isinstance(extra_data, str)

309 if extra_info_type == "00":

310 record["messages"] = parse_record_messages(extra_data)

311 elif extra_info_type == "01":

312 record["counts"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_COUNTS, line_number, record_length=8)

313 elif extra_info_type == "02":

314 record["invoice"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_INVOICE, line_number, record_length=33)

315 elif extra_info_type == "03":

316 record["card"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_CARD, line_number, record_length=34)

317 elif extra_info_type == "04":

318 record["correction"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_CORRECTION, line_number, record_length=18)

319 elif extra_info_type == "05":

320 record["currency"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_CURRENCY, line_number, record_length=41)

321 convert_decimal_fields(record["currency"], TO_FILE_RECORD_EXTRA_INFO_CURRENCY_DECIMALS)

322 elif extra_info_type == "06":

323 record["client_messages"] = parse_record_messages(extra_data)

324 elif extra_info_type == "07":

325 record["bank_messages"] = parse_record_messages(extra_data)

326 elif extra_info_type == "08":

327 record["reason"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_REASON, line_number, record_length=35)

328 elif extra_info_type == "09":

329 record["name_detail"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_NAME_DETAIL, line_number, record_length=35)

330 elif extra_info_type == "11":

331 record["sepa"] = parse_records(extra_data, TO_FILE_RECORD_EXTRA_INFO_SEPA, line_number, record_length=323)

332 else:

333 raise ValidationError(_('Line {line}: Invalid record extra info type "{extra_info_type}"').format(line=line_number, extra_info_type=extra_info_type))

334

335

336 def parse_record_messages(extra_data: str) -> List[str]:

337 msg = []

338 while extra_data:

339 msg.append(extra_data[:35])

340 extra_data = extra_data[35:]

341 return msg