Coverage for jsanctions/un.py: 96%

# Coverage for jsanctions/un.py : 96%

## 142 statements

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1 import logging

2 from typing import Any, Dict, List, Optional

3 from django.core.exceptions import ValidationError

4 from django.db import transaction

5 from django.utils.dateparse import parse_date

6 from django.utils.timezone import now

7 from django.utils.translation import gettext as _

8 from jutil.admin import admin_log

9 from jutil.format import choices_label

10 from jutil.parse import parse_datetime

11 from jutil.xml import xml_to_dict

12

13 from jsanctions.helpers import get_country_iso2_code

14 from jsanctions.models import SanctionsListFile, SanctionEntity, NameAlias, Remark, Address, Identification, \

15 SanctionListObject, SubjectType

16

17 logger = logging.getLogger(__name__)

18

19 UN_LIST_TYPE = "UN"

20

21 UN_XML_ARRAY_TAGS = [

22 'VALUE',

23 'INDIVIDUAL',

24 'INDIVIDUAL_ALIAS',

25 'INDIVIDUAL_ADDRESS',

26 'INDIVIDUAL_DATE_OF_BIRTH',

27 'INDIVIDUAL_PLACE_OF_BIRTH',

28 'INDIVIDUAL_DOCUMENT',

29 'ENTITY',

30 'ENTITY_ALIAS',

31 'ENTITY_ADDRESS',

32]

33

34 UN_NAME_FIELDS = ['FIRST_NAME', 'SECOND_NAME', 'THIRD_NAME', 'FOURTH_NAME', 'FIFTH_NAME', 'SIXTH_NAME']

35

36

37 def load_un_sanction_list_as_dict(filename: str) -> Dict[str, Any]:

38 with open(filename, "rb") as fp:

39 data: Dict[str, Any] = xml_to_dict(fp.read(), array_tags=UN_XML_ARRAY_TAGS)

40 return data

41

42

43 def parse_un_data_id(data: Dict[str, Any]) -> int:

44 uid = data.get('DATAID')

45 if uid is None:

46 raise ValidationError(_('DATAID missing'))

47 return int(uid)

48

49

50 def create_un_alias(se: SanctionEntity, **kwargs) -> Optional[NameAlias]:

51 names = []

52 for k in UN_NAME_FIELDS:

53 if k in kwargs and kwargs[k]:

54 names.append(kwargs[k])

55 if not names:

56 logger.warning('No names: %s', kwargs)

57 return None

58

59 alias = NameAlias(sanction=se, logical_id=parse_un_data_id(kwargs))

60 alias.title = kwargs.get('TITLE') or ''

61 alias.last_name = names.pop() or ''

62 alias.first_name = ' '.join(names).strip()

63 alias.full_clean()

64 alias.save()

65 return alias

66

67

68 def create_un_comments(se: SanctionEntity, **kwargs) -> List[Remark]:

69 out: List[Remark] = []

70 for n in range(1, 10):

71 k = 'COMMENTS{}'.format(n)

72 if k in kwargs and kwargs[k]:

73 obj_out = Remark(container=se, text=kwargs.get(k) or '') # type: ignore

74 obj_out.full_clean()

75 obj_out.save()

76 out.append(obj_out)

77 else:

78 break

79 return out

80

81

82 def create_un_note(obj: SanctionListObject, note: Any):

83 if note:

84 remark = Remark(container=obj, text=str(note))

85 remark.full_clean()

86 remark.save()

87

88

89 def create_un_address(se: SanctionEntity, **kwargs) -> Address:

90 # {'STATE_PROVINCE', 'NOTE', 'COUNTRY', 'STREET', 'CITY', 'ZIP_CODE'}

91 address = Address(sanction=se)

92 address.region = kwargs.get('STATE_PROVINCE') or ''

93 address.city = kwargs.get('CITY') or ''

94 address.zip_code = kwargs.get('ZIP_CODE') or ''

95 address.country_description = kwargs.get('COUNTRY') or ''

96 address.street = kwargs.get('STREET') or ''

97 for k, v in kwargs.items():

98 if hasattr(address, k):

99 setattr(address, k, v)

100 address.full_clean()

101 address.save()

102 create_un_note(address, kwargs.get('NOTE'))

103 return address

104

105

106 def create_un_document(se: SanctionEntity, **kwargs) -> Identification:

107 # {'DATE_OF_ISSUE', 'NUMBER', 'NOTE', 'ISSUING_COUNTRY', 'CITY_OF_ISSUE', 'COUNTRY_OF_ISSUE',

108 # 'TYPE_OF_DOCUMENT', 'TYPE_OF_DOCUMENT2'}

109 id_obj = Identification(sanction=se)

110 id_obj.identification_type_description = kwargs.get('TYPE_OF_DOCUMENT') or kwargs.get('TYPE_OF_DOCUMENT2') or ''

111 id_obj.issue_date = parse_date(str(kwargs.get('DATE_OF_ISSUE'))) if kwargs.get('DATE_OF_ISSUE') else None # type: ignore

112 id_obj.latin_number = kwargs.get('NUMBER') or ''

113 id_obj.issued_by = '{} {} {}'.format(kwargs.get('CITY_OF_ISSUE') or '', kwargs.get('COUNTRY_OF_ISSUE') or '', kwargs.get('ISSUING_COUNTRY') or '').strip()

114 id_obj.country_description = kwargs.get('COUNTRY_OF_ISSUE') or kwargs.get('ISSUING_COUNTRY') or ''

115 id_obj.full_clean()

116 id_obj.save()

117 create_un_note(id_obj, kwargs.get('NOTE'))

118 return id_obj

119

120

121 def set_un_members( # noqa

122 se: SanctionEntity, data: Dict[str, Any], verbose: bool = False, padding: int = 0,

123):

124 # DATAID

125 se.logical_id = parse_un_data_id(data)

126

127 # FIRST_NAME, ...

128 create_un_alias(se, **data)

129

130 # COMMENTSx

131 create_un_comments(se, **data)

132

133 # INVIDUAL_ADDRESS / ENTITY_ADDRESS

134 address_list = data.get("INVIDUAL_ADDRESS", []) or data.get('ENTITY_ADDRESS', [])

135 addresses: List[Address] = []

136 if address_list:

137 for e_data in address_list:

138 if e_data:

139 addresses.append(create_un_address(se, **e_data))

140

141 # try to fill address information from UN list name

142 if not addresses:

143 un_list_type = data.get("UN_LIST_TYPE")

144 if un_list_type:

145 country_code = get_country_iso2_code(un_list_type)

146 if country_code:

147 create_un_address(se, country_description=un_list_type, country_code=country_code)

148

149 # INDIVIDUAL_DOCUMENT

150 docs = data.get('INDIVIDUAL_DOCUMENT')

151 if docs:

152 for e_data in docs:

153 if e_data:

154 create_un_document(se, **e_data)

155

156 se.full_clean()

157 se.save()

158 if verbose:

159 logger.info("%sSaved %s", padding * ' ', se)

160

161

162 def import_un_sanctions(source: SanctionsListFile, verbose: bool = False):

163 data = load_un_sanction_list_as_dict(source.full_path)

164 source.generation_date = parse_datetime(data["@dateGenerated"]).date()

165

166 enterprise, created = SubjectType.objects.get_or_create(classification_code=SubjectType.ENTERPRISE)

167 assert isinstance(enterprise, SubjectType)

168 if created or not enterprise.code:

169 enterprise.code = choices_label(SubjectType.CLASSIFICATION_CODES, enterprise.classification_code)

170 enterprise.save()

171 person, created = SubjectType.objects.get_or_create(classification_code=SubjectType.PERSON)

172 assert isinstance(person, SubjectType)

173 if created or not person.code:

174 person.code = choices_label(SubjectType.CLASSIFICATION_CODES, person.classification_code)

175 person.save()

176

177 t0 = now()

178 individuals_list = data.get("INDIVIDUALS", {}).get('INDIVIDUAL')

179 for se_data in individuals_list:

180 assert isinstance(se_data, dict)

181 if verbose:

182 logger.info(" sdnEntry uid %s", se_data.get('uid'))

183 with transaction.atomic():

184 se = SanctionEntity.objects.create(source=source, data=se_data, subject_type=person)

185 set_un_members(se, se_data, verbose=verbose, padding=4)

186

187 entities_list = data.get("ENTITIES", {}).get('ENTITY')

188 for se_data in entities_list:

189 assert isinstance(se_data, dict)

190 if verbose:

191 logger.info(" sdnEntry uid %s", se_data.get('uid'))

192 with transaction.atomic():

193 se = SanctionEntity.objects.create(source=source, data=se_data, subject_type=enterprise)

194 set_un_members(se, se_data, verbose=verbose, padding=4)

195

196 source.imported = now()

197 source.save()

198 msg = "Imported {} sanction entities and {} individuals from {} in {}".format(

199 len(entities_list), len(individuals_list), source.full_path, source.imported - t0

200 )

201 logger.info(msg)

202 admin_log([source], msg)