Coverage for jsanctions/ofac.py: 95%

# Coverage for jsanctions/ofac.py : 95%

## 162 statements

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1 import logging

2 import re

3 from datetime import date

4 from time import strptime

5 from typing import Dict, Any, Tuple, Optional

6 from django.core.exceptions import ValidationError

7 from django.db import transaction

8 from django.utils import translation

9 from django.utils.timezone import now

10 from django.utils.translation import gettext as _

11 from jutil.admin import admin_log

12 from jutil.format import choices_label

13 from jutil.xml import xml_to_dict

14 from jsanctions.models import SanctionsListFile, SanctionEntity, NameAlias, SubjectType, BirthDate, Address, \

15 Identification, Remark

16

17 logger = logging.getLogger(__name__)

18

19 OFAC_LIST_TYPE = "OFAC"

20

21 OFAC_XML_ARRAY_TAGS = ['sdnEntry', 'program', 'aka', 'dateOfBirthItem', 'placeOfBirthItem', 'address', 'id']

22

23

24 def load_ofac_sanction_list_as_dict(filename: str) -> Dict[str, Any]:

25 with open(filename, "rb") as fp:

26 data: Dict[str, Any] = xml_to_dict(fp.read(), array_tags=OFAC_XML_ARRAY_TAGS)

27 return data

28

29

30 def parse_ofac_date(v: str) -> date:

31 st = strptime(v, '%m/%d/%Y')

32 if not st:

33 raise ValidationError(_('Invalid date value {}').format(v))

34 return date(st.tm_year, st.tm_mon, st.tm_mday)

35

36

37 def parse_ofac_dob(v: str) -> Tuple[Optional[int], Optional[int], Optional[int]]:

38 if re.fullmatch(r'\d{4}', v):

39 return int(v), None, None

40 if re.fullmatch(r'\d{1,2}/\d{1,2}/\d{4}', v):

41 st = strptime(v, '%m/%d/%Y')

42 return st.tm_year, st.tm_mon, st.tm_mday

43 if re.fullmatch(r'\d{1,2} \w{3} \d{4}', v):

44 with translation.override('en_US'):

45 st = strptime(v, '%d %b %Y')

46 return st.tm_year, st.tm_mon, st.tm_mday

47 return None, None, None

48

49

50 def get_opt_ofac_str(data: Dict[str, Any], key: str) -> str:

51 return data.get(key, '') or ''

52

53

54 def get_ofac_subject_type(data: Dict[str, Any]) -> SubjectType:

55 sdn_type = data["sdnType"]

56 if sdn_type == 'Entity':

57 obj, created = SubjectType.objects.get_or_create(classification_code=SubjectType.ENTERPRISE)

58 elif sdn_type == 'Individual':

59 obj, created = SubjectType.objects.get_or_create(classification_code=SubjectType.PERSON)

60 elif sdn_type == 'Vessel':

61 obj, created = SubjectType.objects.get_or_create(classification_code=SubjectType.VESSEL)

62 elif sdn_type == 'Aircraft':

63 obj, created = SubjectType.objects.get_or_create(classification_code=SubjectType.AIRCRAFT)

64 else:

65 logger.warning('Unknown sdnType: %s', sdn_type)

66 obj, created = SubjectType.objects.get_or_create(classification_code=sdn_type, code=sdn_type)

67 assert isinstance(obj, SubjectType)

68 if created:

69 obj.code = choices_label(SubjectType.CLASSIFICATION_CODES, obj.classification_code)

70 obj.save(update_fields=['code'])

71 return obj

72

73

74 def parse_ofac_uid(data: Dict[str, Any]) -> int:

75 uid = data.get('uid')

76 if uid is None:

77 raise ValidationError(_('UID missing'))

78 return int(uid)

79

80

81 def create_ofac_alias(se: SanctionEntity, **kwargs) -> NameAlias:

82 first_name = kwargs.get('firstName') or ''

83 last_name = kwargs.get('lastName') or ''

84 uid = parse_ofac_uid(kwargs)

85 whole_name = (first_name + ' ' + last_name).strip()

86 alias = NameAlias(sanction=se, first_name=first_name, last_name=last_name, whole_name=whole_name, logical_id=uid)

87 alias.full_clean()

88 alias.save()

89 return alias

90

91

92 def create_ofac_dob(se: SanctionEntity, **kwargs) -> BirthDate:

93 dob = BirthDate(sanction=se)

94 dob.logical_id = parse_ofac_uid(kwargs)

95 dob.birth_date_description = kwargs.get("dateOfBirth") or ''

96 year, month_of_year, day_of_month = parse_ofac_dob(dob.birth_date_description)

97 dob.year, dob.month_of_year, dob.day_of_month = year, month_of_year, day_of_month

98 if year and month_of_year and day_of_month:

99 dob.birth_date = date(year, month_of_year, day_of_month)

100 dob.full_clean()

101 dob.save()

102 return dob

103

104

105 def create_ofac_place_of_birth(se: SanctionEntity, **kwargs) -> BirthDate:

106 dob = BirthDate.objects.all().filter(sanction=se, place='').order_by('id').first()

107 if dob is None:

108 dob = BirthDate(sanction=se, logical_id=parse_ofac_uid(kwargs))

109 dob.place = get_opt_ofac_str(kwargs, 'placeOfBirth')

110 dob.full_clean()

111 dob.save()

112 return dob

113

114

115 def create_ofac_address(se: SanctionEntity, **kwargs) -> Address:

116 address = Address(sanction=se)

117 address.logical_id = parse_ofac_uid(kwargs)

118 address.region = get_opt_ofac_str(kwargs, 'stateOrProvince')

119 address.city = get_opt_ofac_str(kwargs, 'city')

120 address.zip_code = get_opt_ofac_str(kwargs, 'postalCode')

121 address.country_description = get_opt_ofac_str(kwargs, 'country')

122 street = ''

123 for n in range(1, 5):

124 k = 'address{}'.format(n)

125 if k in kwargs:

126 street = street + '\n' + kwargs.get(k)

127 else:

128 break

129 address.street = street.strip()

130 address.full_clean()

131 address.save()

132 return address

133

134

135 def create_ofac_id(se: SanctionEntity, **kwargs) -> Identification:

136 id_obj = Identification(sanction=se)

137 id_obj.logical_id = parse_ofac_uid(kwargs)

138 id_obj.number = kwargs.get('idNumber') or ''

139 id_obj.identification_type_description = kwargs.get('idType') or ''

140 id_obj.country_description = kwargs.get('idCountry') or ''

141 id_obj.full_clean()

142 id_obj.save()

143 return id_obj

144

145

146 def set_ofac_members( # noqa

147 se: SanctionEntity, data: Dict[str, Any], verbose: bool = False, padding: int = 0,

148):

149 # uid

150 se.logical_id = parse_ofac_uid(data)

151

152 # firstName, lastName

153 first_name, last_name = get_opt_ofac_str(data, 'firstName'), get_opt_ofac_str(data, 'lastName')

154 if first_name or last_name:

155 create_ofac_alias(se, **data)

156

157 # sdnType

158 se.subject_type = get_ofac_subject_type(data)

159

160 # remarks

161 remarks = data.get('remarks') or ''

162 if remarks:

163 remark_obj = Remark(container=se, text=remarks)

164 remark_obj.full_clean()

165 remark_obj.save()

166

167 # programList

168 for program in data.get('programList', {}).get('program', []) or []:

169 if program:

170 remark_obj = Remark(container=se, text='program={}'.format(program))

171 remark_obj.full_clean()

172 remark_obj.save()

173

174 # akaList

175 for e_data in data.get('akaList', {}).get('aka', []) or []:

176 create_ofac_alias(se, **e_data)

177

178 # dateOfBirthList

179 for e_data in data.get('dateOfBirthList', {}).get('dateOfBirthItem', []) or []:

180 create_ofac_dob(se, **e_data)

181

182 # placeOfBirthList

183 for e_data in data.get('placeOfBirthList', {}).get('placeOfBirthItem', []) or []:

184 create_ofac_place_of_birth(se, **e_data)

185

186 # addressList

187 for e_data in data.get('addressList', {}).get('address', []) or []:

188 create_ofac_address(se, **e_data)

189

190 # idList

191 for e_data in data.get('idList', {}).get('id', []) or []:

192 create_ofac_id(se, **e_data)

193

194 se.full_clean()

195 se.save()

196 if verbose:

197 logger.info("%sSaved %s", padding * ' ', se)

198

199

200 def import_ofac_sanctions(source: SanctionsListFile, verbose: bool = False):

201 data = load_ofac_sanction_list_as_dict(source.full_path)

202 source.generation_date = parse_ofac_date(data['publshInformation']['Publish_Date'])

203

204 t0 = now()

205 entities_list = data.get("sdnEntry", [])

206 for se_data in entities_list:

207 assert isinstance(se_data, dict)

208 if verbose:

209 logger.info(" sdnEntry uid %s", se_data.get('uid'))

210 with transaction.atomic():

211 se = SanctionEntity.objects.create(source=source, data=se_data)

212 set_ofac_members(se, se_data, verbose=verbose, padding=4)

213

214 source.imported = now()

215 source.save()

216 msg = "Imported {} sanction entities from {} in {}".format(

217 len(entities_list), source.full_path, source.imported - t0

218 )

219 logger.info(msg)

220 admin_log([source], msg)