Coverage for /Users/davegaeddert/Development/dropseed/plain/plain-models/plain/models/deletion.py: 45%

1 from collections import Counter, defaultdict

2 from functools import partial, reduce

3 from itertools import chain

4 from operator import attrgetter, or_

5

6 from plain.models import (

7 query_utils,

8 signals,

9 sql,

10 transaction,

11)

12 from plain.models.db import IntegrityError, connections

13 from plain.models.query import QuerySet

14

15

16 class ProtectedError(IntegrityError):

17 def __init__(self, msg, protected_objects):

18 self.protected_objects = protected_objects

19 super().__init__(msg, protected_objects)

20

21

22 class RestrictedError(IntegrityError):

23 def __init__(self, msg, restricted_objects):

24 self.restricted_objects = restricted_objects

25 super().__init__(msg, restricted_objects)

26

27

28 def CASCADE(collector, field, sub_objs, using):

29 collector.collect(

30 sub_objs,

31 source=field.remote_field.model,

32 source_attr=field.name,

33 nullable=field.null,

34 fail_on_restricted=False,

35 )

36 if field.null and not connections[using].features.can_defer_constraint_checks:

37 collector.add_field_update(field, None, sub_objs)

38

39

40 def PROTECT(collector, field, sub_objs, using):

41 raise ProtectedError(

42 "Cannot delete some instances of model '{}' because they are "

43 "referenced through a protected foreign key: '{}.{}'".format(

44 field.remote_field.model.__name__,

45 sub_objs[0].__class__.__name__,

46 field.name,

47 ),

48 sub_objs,

49 )

50

51

52 def RESTRICT(collector, field, sub_objs, using):

53 collector.add_restricted_objects(field, sub_objs)

54 collector.add_dependency(field.remote_field.model, field.model)

55

56

57 def SET(value):

58 if callable(value):

59

60 def set_on_delete(collector, field, sub_objs, using):

61 collector.add_field_update(field, value(), sub_objs)

62

63 else:

64

65 def set_on_delete(collector, field, sub_objs, using):

66 collector.add_field_update(field, value, sub_objs)

67

68 set_on_delete.deconstruct = lambda: ("plain.models.SET", (value,), {})

69 set_on_delete.lazy_sub_objs = True

70 return set_on_delete

71

72

73 def SET_NULL(collector, field, sub_objs, using):

74 collector.add_field_update(field, None, sub_objs)

75

76

77 SET_NULL.lazy_sub_objs = True

78

79

80 def SET_DEFAULT(collector, field, sub_objs, using):

81 collector.add_field_update(field, field.get_default(), sub_objs)

82

83

84 SET_DEFAULT.lazy_sub_objs = True

85

86

87 def DO_NOTHING(collector, field, sub_objs, using):

88 pass

89

90

91 def get_candidate_relations_to_delete(opts):

92 # The candidate relations are the ones that come from N-1 and 1-1 relations.

93 # N-N (i.e., many-to-many) relations aren't candidates for deletion.

94 return (

95 f

96 for f in opts.get_fields(include_hidden=True)

97 if f.auto_created and not f.concrete and (f.one_to_one or f.one_to_many)

98 )

99

100

101 class Collector:

102 def __init__(self, using, origin=None):

103 self.using = using

104 # A Model or QuerySet object.

105 self.origin = origin

106 # Initially, {model: {instances}}, later values become lists.

107 self.data = defaultdict(set)

108 # {(field, value): [instances, …]}

109 self.field_updates = defaultdict(list)

110 # {model: {field: {instances}}}

111 self.restricted_objects = defaultdict(partial(defaultdict, set))

112 # fast_deletes is a list of queryset-likes that can be deleted without

113 # fetching the objects into memory.

114 self.fast_deletes = []

115

116 # Tracks deletion-order dependency for databases without transactions

117 # or ability to defer constraint checks. Only concrete model classes

118 # should be included, as the dependencies exist only between actual

119 # database tables.

120 self.dependencies = defaultdict(set) # {model: {models}}

121

122 def add(self, objs, source=None, nullable=False, reverse_dependency=False):

123 """

124 Add 'objs' to the collection of objects to be deleted. If the call is

125 the result of a cascade, 'source' should be the model that caused it,

126 and 'nullable' should be set to True if the relation can be null.

127

128 Return a list of all objects that were not already collected.

129 """

130 if not objs:

131 return []

132 new_objs = []

133 model = objs[0].__class__

134 instances = self.data[model]

135 for obj in objs:

136 if obj not in instances:

137 new_objs.append(obj)

138 instances.update(new_objs)

139 # Nullable relationships can be ignored -- they are nulled out before

140 # deleting, and therefore do not affect the order in which objects have

141 # to be deleted.

142 if source is not None and not nullable:

143 self.add_dependency(source, model, reverse_dependency=reverse_dependency)

144 return new_objs

145

146 def add_dependency(self, model, dependency, reverse_dependency=False):

147 if reverse_dependency:

148 model, dependency = dependency, model

149 self.dependencies[model._meta.concrete_model].add(

150 dependency._meta.concrete_model

151 )

152 self.data.setdefault(dependency, self.data.default_factory())

153

154 def add_field_update(self, field, value, objs):

155 """

156 Schedule a field update. 'objs' must be a homogeneous iterable

157 collection of model instances (e.g. a QuerySet).

158 """

159 self.field_updates[field, value].append(objs)

160

161 def add_restricted_objects(self, field, objs):

162 if objs:

163 model = objs[0].__class__

164 self.restricted_objects[model][field].update(objs)

165

166 def clear_restricted_objects_from_set(self, model, objs):

167 if model in self.restricted_objects:

168 self.restricted_objects[model] = {

169 field: items - objs

170 for field, items in self.restricted_objects[model].items()

171 }

172

173 def clear_restricted_objects_from_queryset(self, model, qs):

174 if model in self.restricted_objects:

175 objs = set(

176 qs.filter(

177 pk__in=[

178 obj.pk

179 for objs in self.restricted_objects[model].values()

180 for obj in objs

181 ]

182 )

183 )

184 self.clear_restricted_objects_from_set(model, objs)

185

186 def _has_signal_listeners(self, model):

187 return signals.pre_delete.has_listeners(

188 model

189 ) or signals.post_delete.has_listeners(model)

190

191 def can_fast_delete(self, objs, from_field=None):

192 """

193 Determine if the objects in the given queryset-like or single object

194 can be fast-deleted. This can be done if there are no cascades, no

195 parents and no signal listeners for the object class.

196

197 The 'from_field' tells where we are coming from - we need this to

198 determine if the objects are in fact to be deleted. Allow also

199 skipping parent -> child -> parent chain preventing fast delete of

200 the child.

201 """

202 if from_field and from_field.remote_field.on_delete is not CASCADE:

203 return False

204 if hasattr(objs, "_meta"):

205 model = objs._meta.model

206 elif hasattr(objs, "model") and hasattr(objs, "_raw_delete"):

207 model = objs.model

208 else:

209 return False

210 if self._has_signal_listeners(model):

211 return False

212 # The use of from_field comes from the need to avoid cascade back to

213 # parent when parent delete is cascading to child.

214 opts = model._meta

215 return (

216 all(

217 link == from_field

218 for link in opts.concrete_model._meta.parents.values()

219 )

220 and

221 # Foreign keys pointing to this model.

222 all(

223 related.field.remote_field.on_delete is DO_NOTHING

224 for related in get_candidate_relations_to_delete(opts)

225 )

226 and (

227 # Something like generic foreign key.

228 not any(

229 hasattr(field, "bulk_related_objects")

230 for field in opts.private_fields

231 )

232 )

233 )

234

235 def get_del_batches(self, objs, fields):

236 """

237 Return the objs in suitably sized batches for the used connection.

238 """

239 field_names = [field.name for field in fields]

240 conn_batch_size = max(

241 connections[self.using].ops.bulk_batch_size(field_names, objs), 1

242 )

243 if len(objs) > conn_batch_size:

244 return [

245 objs[i : i + conn_batch_size]

246 for i in range(0, len(objs), conn_batch_size)

247 ]

248 else:

249 return [objs]

250

251 def collect(

252 self,

253 objs,

254 source=None,

255 nullable=False,

256 collect_related=True,

257 source_attr=None,

258 reverse_dependency=False,

259 keep_parents=False,

260 fail_on_restricted=True,

261 ):

262 """

263 Add 'objs' to the collection of objects to be deleted as well as all

264 parent instances. 'objs' must be a homogeneous iterable collection of

265 model instances (e.g. a QuerySet). If 'collect_related' is True,

266 related objects will be handled by their respective on_delete handler.

267

268 If the call is the result of a cascade, 'source' should be the model

269 that caused it and 'nullable' should be set to True, if the relation

270 can be null.

271

272 If 'reverse_dependency' is True, 'source' will be deleted before the

273 current model, rather than after. (Needed for cascading to parent

274 models, the one case in which the cascade follows the forwards

275 direction of an FK rather than the reverse direction.)

276

277 If 'keep_parents' is True, data of parent model's will be not deleted.

278

279 If 'fail_on_restricted' is False, error won't be raised even if it's

280 prohibited to delete such objects due to RESTRICT, that defers

281 restricted object checking in recursive calls where the top-level call

282 may need to collect more objects to determine whether restricted ones

283 can be deleted.

284 """

285 if self.can_fast_delete(objs):

286 self.fast_deletes.append(objs)

287 return

288 new_objs = self.add(

289 objs, source, nullable, reverse_dependency=reverse_dependency

290 )

291 if not new_objs:

292 return

293

294 model = new_objs[0].__class__

295

296 if not keep_parents:

297 # Recursively collect concrete model's parent models, but not their

298 # related objects. These will be found by meta.get_fields()

299 concrete_model = model._meta.concrete_model

300 for ptr in concrete_model._meta.parents.values():

301 if ptr:

302 parent_objs = [getattr(obj, ptr.name) for obj in new_objs]

303 self.collect(

304 parent_objs,

305 source=model,

306 source_attr=ptr.remote_field.related_name,

307 collect_related=False,

308 reverse_dependency=True,

309 fail_on_restricted=False,

310 )

311 if not collect_related:

312 return

313

314 if keep_parents:

315 parents = set(model._meta.get_parent_list())

316 model_fast_deletes = defaultdict(list)

317 protected_objects = defaultdict(list)

318 for related in get_candidate_relations_to_delete(model._meta):

319 # Preserve parent reverse relationships if keep_parents=True.

320 if keep_parents and related.model in parents:

321 continue

322 field = related.field

323 on_delete = field.remote_field.on_delete

324 if on_delete == DO_NOTHING:

325 continue

326 related_model = related.related_model

327 if self.can_fast_delete(related_model, from_field=field):

328 model_fast_deletes[related_model].append(field)

329 continue

330 batches = self.get_del_batches(new_objs, [field])

331 for batch in batches:

332 sub_objs = self.related_objects(related_model, [field], batch)

333 # Non-referenced fields can be deferred if no signal receivers

334 # are connected for the related model as they'll never be

335 # exposed to the user. Skip field deferring when some

336 # relationships are select_related as interactions between both

337 # features are hard to get right. This should only happen in

338 # the rare cases where .related_objects is overridden anyway.

339 if not (

340 sub_objs.query.select_related

341 or self._has_signal_listeners(related_model)

342 ):

343 referenced_fields = set(

344 chain.from_iterable(

345 (rf.attname for rf in rel.field.foreign_related_fields)

346 for rel in get_candidate_relations_to_delete(

347 related_model._meta

348 )

349 )

350 )

351 sub_objs = sub_objs.only(*tuple(referenced_fields))

352 if getattr(on_delete, "lazy_sub_objs", False) or sub_objs:

353 try:

354 on_delete(self, field, sub_objs, self.using)

355 except ProtectedError as error:

356 key = f"'{field.model.__name__}.{field.name}'"

357 protected_objects[key] += error.protected_objects

358 if protected_objects:

359 raise ProtectedError(

360 "Cannot delete some instances of model {!r} because they are "

361 "referenced through protected foreign keys: {}.".format(

362 model.__name__,

363 ", ".join(protected_objects),

364 ),

365 set(chain.from_iterable(protected_objects.values())),

366 )

367 for related_model, related_fields in model_fast_deletes.items():

368 batches = self.get_del_batches(new_objs, related_fields)

369 for batch in batches:

370 sub_objs = self.related_objects(related_model, related_fields, batch)

371 self.fast_deletes.append(sub_objs)

372 for field in model._meta.private_fields:

373 if hasattr(field, "bulk_related_objects"):

374 # It's something like generic foreign key.

375 sub_objs = field.bulk_related_objects(new_objs, self.using)

376 self.collect(

377 sub_objs, source=model, nullable=True, fail_on_restricted=False

378 )

379

380 if fail_on_restricted:

381 # Raise an error if collected restricted objects (RESTRICT) aren't

382 # candidates for deletion also collected via CASCADE.

383 for related_model, instances in self.data.items():

384 self.clear_restricted_objects_from_set(related_model, instances)

385 for qs in self.fast_deletes:

386 self.clear_restricted_objects_from_queryset(qs.model, qs)

387 if self.restricted_objects.values():

388 restricted_objects = defaultdict(list)

389 for related_model, fields in self.restricted_objects.items():

390 for field, objs in fields.items():

391 if objs:

392 key = f"'{related_model.__name__}.{field.name}'"

393 restricted_objects[key] += objs

394 if restricted_objects:

395 raise RestrictedError(

396 "Cannot delete some instances of model {!r} because "

397 "they are referenced through restricted foreign keys: "

398 "{}.".format(

399 model.__name__,

400 ", ".join(restricted_objects),

401 ),

402 set(chain.from_iterable(restricted_objects.values())),

403 )

404

405 def related_objects(self, related_model, related_fields, objs):

406 """

407 Get a QuerySet of the related model to objs via related fields.

408 """

409 predicate = query_utils.Q.create(

410 [(f"{related_field.name}__in", objs) for related_field in related_fields],

411 connector=query_utils.Q.OR,

412 )

413 return related_model._base_manager.using(self.using).filter(predicate)

414

415 def instances_with_model(self):

416 for model, instances in self.data.items():

417 for obj in instances:

418 yield model, obj

419

420 def sort(self):

421 sorted_models = []

422 concrete_models = set()

423 models = list(self.data)

424 while len(sorted_models) < len(models):

425 found = False

426 for model in models:

427 if model in sorted_models:

428 continue

429 dependencies = self.dependencies.get(model._meta.concrete_model)

430 if not (dependencies and dependencies.difference(concrete_models)):

431 sorted_models.append(model)

432 concrete_models.add(model._meta.concrete_model)

433 found = True

434 if not found:

435 return

436 self.data = {model: self.data[model] for model in sorted_models}

437

438 def delete(self):

439 # sort instance collections

440 for model, instances in self.data.items():

441 self.data[model] = sorted(instances, key=attrgetter("pk"))

442

443 # if possible, bring the models in an order suitable for databases that

444 # don't support transactions or cannot defer constraint checks until the

445 # end of a transaction.

446 self.sort()

447 # number of objects deleted for each model label

448 deleted_counter = Counter()

449

450 # Optimize for the case with a single obj and no dependencies

451 if len(self.data) == 1 and len(instances) == 1:

452 instance = list(instances)[0]

453 if self.can_fast_delete(instance):

454 with transaction.mark_for_rollback_on_error(self.using):

455 count = sql.DeleteQuery(model).delete_batch(

456 [instance.pk], self.using

457 )

458 setattr(instance, model._meta.pk.attname, None)

459 return count, {model._meta.label: count}

460

461 with transaction.atomic(using=self.using, savepoint=False):

462 # send pre_delete signals

463 for model, obj in self.instances_with_model():

464 if not model._meta.auto_created:

465 signals.pre_delete.send(

466 sender=model,

467 instance=obj,

468 using=self.using,

469 origin=self.origin,

470 )

471

472 # fast deletes

473 for qs in self.fast_deletes:

474 count = qs._raw_delete(using=self.using)

475 if count:

476 deleted_counter[qs.model._meta.label] += count

477

478 # update fields

479 for (field, value), instances_list in self.field_updates.items():

480 updates = []

481 objs = []

482 for instances in instances_list:

483 if (

484 isinstance(instances, QuerySet)

485 and instances._result_cache is None

486 ):

487 updates.append(instances)

488 else:

489 objs.extend(instances)

490 if updates:

491 combined_updates = reduce(or_, updates)

492 combined_updates.update(**{field.name: value})

493 if objs:

494 model = objs[0].__class__

495 query = sql.UpdateQuery(model)

496 query.update_batch(

497 list({obj.pk for obj in objs}), {field.name: value}, self.using

498 )

499

500 # reverse instance collections

501 for instances in self.data.values():

502 instances.reverse()

503

504 # delete instances

505 for model, instances in self.data.items():

506 query = sql.DeleteQuery(model)

507 pk_list = [obj.pk for obj in instances]

508 count = query.delete_batch(pk_list, self.using)

509 if count:

510 deleted_counter[model._meta.label] += count

511

512 if not model._meta.auto_created:

513 for obj in instances:

514 signals.post_delete.send(

515 sender=model,

516 instance=obj,

517 using=self.using,

518 origin=self.origin,

519 )

520

521 for model, instances in self.data.items():

522 for instance in instances:

523 setattr(instance, model._meta.pk.attname, None)

524 return sum(deleted_counter.values()), dict(deleted_counter)