Coverage for jsanctions/eu.py: 74%

# Coverage for jsanctions/eu.py : 74%

## 104 statements

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1 from typing import List, Dict

2 from jsanctions.helpers import dict_filter_attributes

3 from jutil.parse import parse_datetime

4 from jutil.xml import xml_to_dict

5 import logging

6 import os

7 from typing import Any

8 from django.db import transaction

9 from django.utils.timezone import now

10 from jutil.admin import admin_log

11 from jsanctions.models import (

12 SanctionsListFile,

13 SanctionEntity,

14 RegulationSummary,

15 BirthDate,

16 Identification,

17 Remark,

18 Address,

19 Citizenship,

20 NameAlias,

21 Regulation,

22 SubjectType,

23)

24 from jutil.format import camel_case_to_underscore

25

26 logger = logging.getLogger(__name__)

27

28 EU_LIST_TYPE = "EU"

29

30 EU_XML_ARRAY_TAGS = [

31 "sanctionEntity",

32 "nameAlias",

33 "identification",

34 "address",

35 "birthdate",

36 "citizenship",

37 "regulation",

38 "remark",

39]

40

41 EU_XML_INT_TAGS: List[str] = []

42

43 EU_XML_DATE_ATTRIBUTES = [

44 "@birthdate",

45]

46

47

48 def eu_sanction_list_xml_attr_filter(k: str, v: Any) -> Any:

49 if k.endswith("Date") or k in EU_XML_DATE_ATTRIBUTES:

50 return parse_datetime(v).date()

51 if k == "@logicalId":

52 return int(v)

53 if v == "false":

54 return False

55 if v == "true":

56 return True

57 return v

58

59

60 def load_eu_sanction_list_as_dict(filename: str) -> Dict[str, Any]:

61 with open(filename, "rb") as fp:

62 data: Dict[str, Any] = xml_to_dict(fp.read(), array_tags=EU_XML_ARRAY_TAGS, int_tags=EU_XML_INT_TAGS)

63 data = dict_filter_attributes(data, eu_sanction_list_xml_attr_filter)

64 return data

65

66

67 def set_eu_object_attr(obj, k: str, v, max_length: int = 512):

68 if v and isinstance(v, str) and len(v) > max_length:

69 logger.warning("'%s' truncated to [%s]: '%s...'", k, max_length, v[:64])

70 v = v[: max_length - 3] + "..."

71 setattr(obj, k, v)

72

73

74 def set_eu_members( # noqa

75 obj: Any, data: Dict[str, Any], verbose: bool = False, padding: int = 0, **kwargs

76):

77 class_map = {

78 "regulationSummary": RegulationSummary,

79 "subjectType": SubjectType,

80 }

81 array_class_map = {

82 "birthdate": BirthDate,

83 "identification": Identification,

84 "address": Address,

85 "citizenship": Citizenship,

86 "nameAlias": NameAlias,

87 "regulation": Regulation,

88 }

89

90 padding_str = " " * padding

91 obj.save()

92 obj2: Any

93 for k0, v0 in data.items():

94 if k0[0] == "@":

95 k = camel_case_to_underscore(k0[1:])

96 if k == "birthdate":

97 k = "birth_date" # special case because class name same as attribute

98 if hasattr(obj, k):

99 set_eu_object_attr(obj, k, v0)

100 if verbose:

101 logger.info("%s%s: %s = %s", padding_str, obj, k, v0)

102 elif k0 in class_map:

103 k = camel_case_to_underscore(k0)

104 if hasattr(obj, k):

105 if k == "subject_type":

106 obj2 = SubjectType.objects.get_or_create(

107 code=v0.get("@code", ""), classification_code=v0.get("@classificationCode", "")

108 )[0]

109 elif k == "regulation_summary":

110 obj2 = RegulationSummary.objects.get_or_create(

111 regulation_type=v0.get("@regulationType", ""),

112 publication_date=v0.get("@publicationDate", None),

113 publication_url=v0.get("@publicationUrl", ""),

114 number_title=v0.get("@numberTitle", ""),

115 )[0]

116 else:

117 obj2 = class_map[k0]()

118 kwargs2 = {}

119 kwargs2[k] = obj2

120 for k2, v2 in kwargs.items():

121 set_eu_object_attr(obj2, k2, v2)

122 kwargs2[k2] = v2

123

124 set_eu_members(obj2, v0, verbose=verbose, padding=padding + 4, **kwargs2)

125

126 set_eu_object_attr(obj, k, obj2)

127 elif k0 in array_class_map:

128 k = camel_case_to_underscore(k0)

129 cls = array_class_map[k0]

130 for v0_data in v0:

131 obj2 = cls()

132 kwargs2 = {}

133 kwargs2[k] = obj2

134 for k2, v2 in kwargs.items():

135 set_eu_object_attr(obj2, k2, v2)

136 kwargs2[k2] = v2

137 obj2.clean()

138 obj2.save()

139 set_eu_members(obj2, v0_data, verbose=verbose, padding=padding + 4, **kwargs2)

140 elif k0 == "remark":

141 for v0_str in v0:

142 Remark.objects.create(text=v0_str, container=obj)

143

144 obj.clean()

145 obj.save()

146 if verbose:

147 logger.info("%sSaved %s", padding * ' ', obj)

148

149

150 def import_eu_sanctions(source: SanctionsListFile, verbose: bool = False):

151 data = load_eu_sanction_list_as_dict(source.full_path)

152 set_eu_members(source, data, verbose=verbose)

153 entities_list = data.get("sanctionEntity", [])

154 logger.info("Importing %s sanction entities from %s", len(entities_list), os.path.basename(source.file.name))

155 t0 = now()

156 for se_data in entities_list:

157 assert isinstance(se_data, dict)

158 if verbose:

159 logger.info(" sanctionEntity")

160 with transaction.atomic():

161 se = SanctionEntity.objects.create(source=source, data=se_data)

162 set_eu_members(se, se_data, verbose=verbose, padding=4, sanction=se)

163 source.imported = now()

164 source.save()

165 msg = "Imported {} sanction entities from {} in {}".format(

166 len(entities_list), source.full_path, source.imported - t0

167 )

168 logger.info(msg)

169 admin_log([source], msg)