Coverage for /Users/davegaeddert/Developer/dropseed/plain/plain-models/plain/models/query.py: 27%

1"""

2 The main QuerySet implementation. This provides the public API for the ORM.

3"""

4

5 import copy

6 import operator

7 import warnings

8 from itertools import chain, islice

9

10 import plain.runtime

11 from plain import exceptions

12 from plain.exceptions import ValidationError

13 from plain.models import (

14 sql,

15 transaction,

16)

17 from plain.models.constants import LOOKUP_SEP, OnConflict

18 from plain.models.db import (

19 PLAIN_VERSION_PICKLE_KEY,

20 IntegrityError,

21 NotSupportedError,

22 connections,

23 router,

24)

25 from plain.models.expressions import Case, F, Value, When

26 from plain.models.fields import (

27 AutoField,

28 DateField,

29 DateTimeField,

30 Field,

31)

32 from plain.models.functions import Cast, Trunc

33 from plain.models.query_utils import FilteredRelation, Q

34 from plain.models.sql.constants import CURSOR, GET_ITERATOR_CHUNK_SIZE

35 from plain.models.utils import (

36 AltersData,

37 create_namedtuple_class,

38 resolve_callables,

39)

40 from plain.utils import timezone

41 from plain.utils.functional import cached_property, partition

42

43# The maximum number of results to fetch in a get() query.

44 MAX_GET_RESULTS = 21

45

46# The maximum number of items to display in a QuerySet.__repr__

47 REPR_OUTPUT_SIZE = 20

48

49

50 class BaseIterable:

51 def __init__(

52 self, queryset, chunked_fetch=False, chunk_size=GET_ITERATOR_CHUNK_SIZE

53 ):

54 self.queryset = queryset

55 self.chunked_fetch = chunked_fetch

56 self.chunk_size = chunk_size

57

58

59 class ModelIterable(BaseIterable):

60 """Iterable that yields a model instance for each row."""

61

62 def __iter__(self):

63 queryset = self.queryset

64 db = queryset.db

65 compiler = queryset.query.get_compiler(using=db)

66 # Execute the query. This will also fill compiler.select, klass_info,

67 # and annotations.

68 results = compiler.execute_sql(

69 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size

70 )

71 select, klass_info, annotation_col_map = (

72 compiler.select,

73 compiler.klass_info,

74 compiler.annotation_col_map,

75 )

76 model_cls = klass_info["model"]

77 select_fields = klass_info["select_fields"]

78 model_fields_start, model_fields_end = select_fields[0], select_fields[-1] + 1

79 init_list = [

80 f[0].target.attname for f in select[model_fields_start: model_fields_end]

81 ]

82 related_populators = get_related_populators(klass_info, select, db)

83 known_related_objects = [

84 (

85 field,

86 related_objs,

87 operator.attrgetter(

88 *[

89 field.attname

90 if from_field == "self"

91 else queryset.model._meta.get_field(from_field).attname

92 for from_field in field.from_fields

93 ]

94 ),

95 )

96 for field, related_objs in queryset._known_related_objects.items()

97 ]

98 for row in compiler.results_iter(results):

99 obj = model_cls.from_db(

100 db, init_list, row[model_fields_start: model_fields_end]

101 )

102 for rel_populator in related_populators:

103 rel_populator.populate(row, obj)

104 if annotation_col_map:

105 for attr_name, col_pos in annotation_col_map.items():

106 setattr(obj, attr_name, row[col_pos])

107

108 # Add the known related objects to the model.

109 for field, rel_objs, rel_getter in known_related_objects:

110 # Avoid overwriting objects loaded by, e.g., select_related().

111 if field.is_cached(obj):

112 continue

113 rel_obj_id = rel_getter(obj)

114 try:

115 rel_obj = rel_objs[rel_obj_id]

116 except KeyError:

117 pass # May happen in qs1 | qs2 scenarios.

118 else:

119 setattr(obj, field.name, rel_obj)

120

121 yield obj

122

123

124 class RawModelIterable(BaseIterable):

125 """

126 Iterable that yields a model instance for each row from a raw queryset.

127 """

128

129 def __iter__(self):

130 # Cache some things for performance reasons outside the loop.

131 db = self.queryset.db

132 query = self.queryset.query

133 connection = connections[db]

134 compiler = connection.ops.compiler("SQLCompiler")(query, connection, db)

135 query_iterator = iter(query)

136

137 try:

138 (

139 model_init_names,

140 model_init_pos,

141 annotation_fields,

142 ) = self.queryset.resolve_model_init_order()

143 model_cls = self.queryset.model

144 if model_cls._meta.pk.attname not in model_init_names:

145 raise exceptions.FieldDoesNotExist(

146 "Raw query must include the primary key"

147 )

148 fields = [self.queryset.model_fields.get(c) for c in self.queryset.columns]

149 converters = compiler.get_converters(

150 [f.get_col(f.model._meta.db_table) if f else None for f in fields]

151 )

152 if converters:

153 query_iterator = compiler.apply_converters(query_iterator, converters)

154 for values in query_iterator:

155 # Associate fields to values

156 model_init_values = [values[pos] for pos in model_init_pos]

157 instance = model_cls.from_db(db, model_init_names, model_init_values)

158 if annotation_fields:

159 for column, pos in annotation_fields:

160 setattr(instance, column, values[pos])

161 yield instance

162 finally:

163 # Done iterating the Query. If it has its own cursor, close it.

164 if hasattr(query, "cursor") and query.cursor:

165 query.cursor.close()

166

167

168 class ValuesIterable(BaseIterable):

169 """

170 Iterable returned by QuerySet.values() that yields a dict for each row.

171 """

172

173 def __iter__(self):

174 queryset = self.queryset

175 query = queryset.query

176 compiler = query.get_compiler(queryset.db)

177

178 # extra(select=...) cols are always at the start of the row.

179 names = [

180 *query.extra_select,

181 *query.values_select,

182 *query.annotation_select,

183 ]

184 indexes = range(len(names))

185 for row in compiler.results_iter(

186 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size

187 ):

188 yield {names[i]: row[i] for i in indexes}

189

190

191 class ValuesListIterable(BaseIterable):

192 """

193 Iterable returned by QuerySet.values_list(flat=False) that yields a tuple

194 for each row.

195 """

196

197 def __iter__(self):

198 queryset = self.queryset

199 query = queryset.query

200 compiler = query.get_compiler(queryset.db)

201

202 if queryset._fields:

203 # extra(select=...) cols are always at the start of the row.

204 names = [

205 *query.extra_select,

206 *query.values_select,

207 *query.annotation_select,

208 ]

209 fields = [

210 *queryset._fields,

211 *(f for f in query.annotation_select if f not in queryset._fields),

212 ]

213 if fields != names:

214 # Reorder according to fields.

215 index_map = {name: idx for idx, name in enumerate(names)}

216 rowfactory = operator.itemgetter(*[index_map[f] for f in fields])

217 return map(

218 rowfactory,

219 compiler.results_iter(

220 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size

221 ),

222 )

223 return compiler.results_iter(

224 tuple_expected=True,

225 chunked_fetch=self.chunked_fetch,

226 chunk_size=self.chunk_size,

227 )

228

229

230 class NamedValuesListIterable(ValuesListIterable):

231 """

232 Iterable returned by QuerySet.values_list(named=True) that yields a

233 namedtuple for each row.

234 """

235

236 def __iter__(self):

237 queryset = self.queryset

238 if queryset._fields:

239 names = queryset._fields

240 else:

241 query = queryset.query

242 names = [

243 *query.extra_select,

244 *query.values_select,

245 *query.annotation_select,

246 ]

247 tuple_class = create_namedtuple_class(*names)

248 new = tuple.__new__

249 for row in super().__iter__():

250 yield new(tuple_class, row)

251

252

253 class FlatValuesListIterable(BaseIterable):

254 """

255 Iterable returned by QuerySet.values_list(flat=True) that yields single

256 values.

257 """

258

259 def __iter__(self):

260 queryset = self.queryset

261 compiler = queryset.query.get_compiler(queryset.db)

262 for row in compiler.results_iter(

263 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size

264 ):

265 yield row[0]

266

267

268 class QuerySet(AltersData):

269 """Represent a lazy database lookup for a set of objects."""

270

271 def __init__(self, model=None, query=None, using=None, hints=None):

272 self.model = model

273 self._db = using

274 self._hints = hints or {}

275 self._query = query or sql.Query(self.model)

276 self._result_cache = None

277 self._sticky_filter = False

278 self._for_write = False

279 self._prefetch_related_lookups = ()

280 self._prefetch_done = False

281 self._known_related_objects = {} # {rel_field: {pk: rel_obj}}

282 self._iterable_class = ModelIterable

283 self._fields = None

284 self._defer_next_filter = False

285 self._deferred_filter = None

286

287 @property

288 def query(self):

289 if self._deferred_filter:

290 negate, args, kwargs = self._deferred_filter

291 self._filter_or_exclude_inplace(negate, args, kwargs)

292 self._deferred_filter = None

293 return self._query

294

295 @query.setter

296 def query(self, value):

297 if value.values_select:

298 self._iterable_class = ValuesIterable

299 self._query = value

300

301 def as_manager(cls):

302 # Address the circular dependency between `Queryset` and `Manager`.

303 from plain.models.manager import Manager

304

305 manager = Manager.from_queryset(cls)()

306 manager._built_with_as_manager = True

307 return manager

308

309 as_manager.queryset_only = True

310 as_manager = classmethod(as_manager)

311

312 ########################

313 # PYTHON MAGIC METHODS #

314 ########################

315

316 def __deepcopy__(self, memo):

317 """Don't populate the QuerySet's cache."""

318 obj = self.__class__()

319 for k, v in self.__dict__.items():

320 if k == "_result_cache":

321 obj.__dict__[k] = None

322 else:

323 obj.__dict__[k] = copy.deepcopy(v, memo)

324 return obj

325

326 def __getstate__(self):

327 # Force the cache to be fully populated.

328 self._fetch_all()

329 return {**self.__dict__, PLAIN_VERSION_PICKLE_KEY: plain.runtime.__version__}

330

331 def __setstate__(self, state):

332 pickled_version = state.get(PLAIN_VERSION_PICKLE_KEY)

333 if pickled_version:

334 if pickled_version != plain.runtime.__version__:

335 warnings.warn(

336 f"Pickled queryset instance's Plain version {pickled_version} does not "

337 f"match the current version {plain.runtime.__version__}.",

338 RuntimeWarning,

339 stacklevel=2,

340 )

341 else:

342 warnings.warn(

343 "Pickled queryset instance's Plain version is not specified.",

344 RuntimeWarning,

345 stacklevel=2,

346 )

347 self.__dict__.update(state)

348

349 def __repr__(self):

350 data = list(self[: REPR_OUTPUT_SIZE + 1])

351 if len(data) > REPR_OUTPUT_SIZE:

352 data[-1] = "...(remaining elements truncated)..."

353 return f"<{self.__class__.__name__} {data! r}>"

354

355 def __len__(self):

356 self._fetch_all()

357 return len(self._result_cache)

358

359 def __iter__(self):

360 """

361 The queryset iterator protocol uses three nested iterators in the

362 default case:

363 1. sql.compiler.execute_sql()

364 - Returns 100 rows at time (constants.GET_ITERATOR_CHUNK_SIZE)

365 using cursor.fetchmany(). This part is responsible for

366 doing some column masking, and returning the rows in chunks.

367 2. sql.compiler.results_iter()

368 - Returns one row at time. At this point the rows are still just

369 tuples. In some cases the return values are converted to

370 Python values at this location.

371 3. self.iterator()

372 - Responsible for turning the rows into model objects.

373 """

374 self._fetch_all()

375 return iter(self._result_cache)

376

377 def __bool__(self):

378 self._fetch_all()

379 return bool(self._result_cache)

380

381 def __getitem__(self, k):

382 """Retrieve an item or slice from the set of results."""

383 if not isinstance(k, int | slice):

384 raise TypeError(

385 f"QuerySet indices must be integers or slices, not {type(k).__name__}."

386 )

387 if (isinstance(k, int) and k < 0) or (

388 isinstance(k, slice)

389 and (

390 (k.start is not None and k.start < 0)

391 or (k.stop is not None and k.stop < 0)

392 )

393 ):

394 raise ValueError("Negative indexing is not supported.")

395

396 if self._result_cache is not None:

397 return self._result_cache[k]

398

399 if isinstance(k, slice):

400 qs = self._chain()

401 if k.start is not None:

402 start = int(k.start)

403 else:

404 start = None

405 if k.stop is not None:

406 stop = int(k.stop)

407 else:

408 stop = None

409 qs.query.set_limits(start, stop)

410 return list(qs)[:: k.step] if k.step else qs

411

412 qs = self._chain()

413 qs.query.set_limits(k, k + 1)

414 qs._fetch_all()

415 return qs._result_cache[0]

416

417 def __class_getitem__(cls, *args, **kwargs):

418 return cls

419

420 def __and__(self, other):

421 self._check_operator_queryset(other, "&")

422 self._merge_sanity_check(other)

423 if isinstance(other, EmptyQuerySet):

424 return other

425 if isinstance(self, EmptyQuerySet):

426 return self

427 combined = self._chain()

428 combined._merge_known_related_objects(other)

429 combined.query.combine(other.query, sql.AND)

430 return combined

431

432 def __or__(self, other):

433 self._check_operator_queryset(other, "|")

434 self._merge_sanity_check(other)

435 if isinstance(self, EmptyQuerySet):

436 return other

437 if isinstance(other, EmptyQuerySet):

438 return self

439 query = (

440 self

441 if self.query.can_filter()

442 else self.model._base_manager.filter(pk__in=self.values("pk"))

443 )

444 combined = query._chain()

445 combined._merge_known_related_objects(other)

446 if not other.query.can_filter():

447 other = other.model._base_manager.filter(pk__in=other.values("pk"))

448 combined.query.combine(other.query, sql.OR)

449 return combined

450

451 def __xor__(self, other):

452 self._check_operator_queryset(other, "^")

453 self._merge_sanity_check(other)

454 if isinstance(self, EmptyQuerySet):

455 return other

456 if isinstance(other, EmptyQuerySet):

457 return self

458 query = (

459 self

460 if self.query.can_filter()

461 else self.model._base_manager.filter(pk__in=self.values("pk"))

462 )

463 combined = query._chain()

464 combined._merge_known_related_objects(other)

465 if not other.query.can_filter():

466 other = other.model._base_manager.filter(pk__in=other.values("pk"))

467 combined.query.combine(other.query, sql.XOR)

468 return combined

469

470 ####################################

471 # METHODS THAT DO DATABASE QUERIES #

472 ####################################

473

474 def _iterator(self, use_chunked_fetch, chunk_size):

475 iterable = self._iterable_class(

476 self,

477 chunked_fetch=use_chunked_fetch,

478 chunk_size=chunk_size or 2000,

479 )

480 if not self._prefetch_related_lookups or chunk_size is None:

481 yield from iterable

482 return

483

484 iterator = iter(iterable)

485 while results := list(islice(iterator, chunk_size)):

486 prefetch_related_objects(results, *self._prefetch_related_lookups)

487 yield from results

488

489 def iterator(self, chunk_size=None):

490 """

491 An iterator over the results from applying this QuerySet to the

492 database. chunk_size must be provided for QuerySets that prefetch

493 related objects. Otherwise, a default chunk_size of 2000 is supplied.

494 """

495 if chunk_size is None:

496 if self._prefetch_related_lookups:

497 raise ValueError(

498 "chunk_size must be provided when using QuerySet.iterator() after "

499 "prefetch_related()."

500 )

501 elif chunk_size <= 0:

502 raise ValueError("Chunk size must be strictly positive.")

503 use_chunked_fetch = not connections[self.db].settings_dict.get(

504 "DISABLE_SERVER_SIDE_CURSORS"

505 )

506 return self._iterator(use_chunked_fetch, chunk_size)

507

508 def aggregate(self, *args, **kwargs):

509 """

510 Return a dictionary containing the calculations (aggregation)

511 over the current queryset.

512

513 If args is present the expression is passed as a kwarg using

514 the Aggregate object's default alias.

515 """

516 if self.query.distinct_fields:

517 raise NotImplementedError("aggregate() + distinct(fields) not implemented.")

518 self._validate_values_are_expressions(

519 (*args, *kwargs.values()), method_name="aggregate"

520 )

521 for arg in args:

522 # The default_alias property raises TypeError if default_alias

523 # can't be set automatically or AttributeError if it isn't an

524 # attribute.

525 try:

526 arg.default_alias

527 except (AttributeError, TypeError):

528 raise TypeError("Complex aggregates require an alias")

529 kwargs[arg.default_alias] = arg

530

531 return self.query.chain().get_aggregation(self.db, kwargs)

532

533 def count(self):

534 """

535 Perform a SELECT COUNT() and return the number of records as an

536 integer.

537

538 If the QuerySet is already fully cached, return the length of the

539 cached results set to avoid multiple SELECT COUNT(*) calls.

540 """

541 if self._result_cache is not None:

542 return len(self._result_cache)

543

544 return self.query.get_count(using=self.db)

545

546 def get(self, *args, **kwargs):

547 """

548 Perform the query and return a single object matching the given

549 keyword arguments.

550 """

551 if self.query.combinator and (args or kwargs):

552 raise NotSupportedError(

553 f"Calling QuerySet.get(...) with filters after {self.query.combinator}() is not "

554 "supported."

555 )

556 clone = self._chain() if self.query.combinator else self.filter(*args, **kwargs)

557 if self.query.can_filter() and not self.query.distinct_fields:

558 clone = clone.order_by()

559 limit = None

560 if (

561 not clone.query.select_for_update

562 or connections[clone.db].features.supports_select_for_update_with_limit

563 ):

564 limit = MAX_GET_RESULTS

565 clone.query.set_limits(high=limit)

566 num = len(clone)

567 if num == 1:

568 return clone._result_cache[0]

569 if not num:

570 raise self.model.DoesNotExist(

571 f"{self.model._meta.object_name} matching query does not exist."

572 )

573 raise self.model.MultipleObjectsReturned(

574 "get() returned more than one {} -- it returned {}!".format(

575 self.model._meta.object_name,

576 num if not limit or num < limit else "more than %s" % (limit - 1),

577 )

578 )

579

580 def create(self, **kwargs):

581 """

582 Create a new object with the given kwargs, saving it to the database

583 and returning the created object.

584 """

585 obj = self.model(**kwargs)

586 self._for_write = True

587 obj.save(force_insert=True, using=self.db)

588 return obj

589

590 def _prepare_for_bulk_create(self, objs):

591 for obj in objs:

592 if obj.pk is None:

593 # Populate new PK values.

594 obj.pk = obj._meta.pk.get_pk_value_on_save(obj)

595 obj._prepare_related_fields_for_save(operation_name="bulk_create")

596

597 def _check_bulk_create_options(

598 self, ignore_conflicts, update_conflicts, update_fields, unique_fields

599 ):

600 if ignore_conflicts and update_conflicts:

601 raise ValueError(

602 "ignore_conflicts and update_conflicts are mutually exclusive."

603 )

604 db_features = connections[self.db].features

605 if ignore_conflicts:

606 if not db_features.supports_ignore_conflicts:

607 raise NotSupportedError(

608 "This database backend does not support ignoring conflicts."

609 )

610 return OnConflict.IGNORE

611 elif update_conflicts:

612 if not db_features.supports_update_conflicts:

613 raise NotSupportedError(

614 "This database backend does not support updating conflicts."

615 )

616 if not update_fields:

617 raise ValueError(

618 "Fields that will be updated when a row insertion fails "

619 "on conflicts must be provided."

620 )

621 if unique_fields and not db_features.supports_update_conflicts_with_target:

622 raise NotSupportedError(

623 "This database backend does not support updating "

624 "conflicts with specifying unique fields that can trigger "

625 "the upsert."

626 )

627 if not unique_fields and db_features.supports_update_conflicts_with_target:

628 raise ValueError(

629 "Unique fields that can trigger the upsert must be provided."

630 )

631 # Updating primary keys and non-concrete fields is forbidden.

632 if any(not f.concrete or f.many_to_many for f in update_fields):

633 raise ValueError(

634 "bulk_create() can only be used with concrete fields in "

635 "update_fields."

636 )

637 if any(f.primary_key for f in update_fields):

638 raise ValueError(

639 "bulk_create() cannot be used with primary keys in "

640 "update_fields."

641 )

642 if unique_fields:

643 if any(not f.concrete or f.many_to_many for f in unique_fields):

644 raise ValueError(

645 "bulk_create() can only be used with concrete fields "

646 "in unique_fields."

647 )

648 return OnConflict.UPDATE

649 return None

650

651 def bulk_create(

652 self,

653 objs,

654 batch_size=None,

655 ignore_conflicts=False,

656 update_conflicts=False,

657 update_fields=None,

658 unique_fields=None,

659 ):

660 """

661 Insert each of the instances into the database. Do *not* call

662 save() on each of the instances, do not send any pre/post_save

663 signals, and do not set the primary key attribute if it is an

664 autoincrement field (except if features.can_return_rows_from_bulk_insert=True).

665 Multi-table models are not supported.

666 """

667 # When you bulk insert you don't get the primary keys back (if it's an

668 # autoincrement, except if can_return_rows_from_bulk_insert=True), so

669 # you can't insert into the child tables which references this. There

670 # are two workarounds:

671 # 1) This could be implemented if you didn't have an autoincrement pk

672 # 2) You could do it by doing O(n) normal inserts into the parent

673 # tables to get the primary keys back and then doing a single bulk

674 # insert into the childmost table.

675 # We currently set the primary keys on the objects when using

676 # PostgreSQL via the RETURNING ID clause. It should be possible for

677 # Oracle as well, but the semantics for extracting the primary keys is

678 # trickier so it's not done yet.

679 if batch_size is not None and batch_size <= 0:

680 raise ValueError("Batch size must be a positive integer.")

681 # Check that the parents share the same concrete model with the our

682 # model to detect the inheritance pattern ConcreteGrandParent ->

683 # MultiTableParent -> ProxyChild. Simply checking self.model._meta.proxy

684 # would not identify that case as involving multiple tables.

685 for parent in self.model._meta.get_parent_list():

686 if parent._meta.concrete_model is not self.model._meta.concrete_model:

687 raise ValueError("Can't bulk create a multi-table inherited model")

688 if not objs:

689 return objs

690 opts = self.model._meta

691 if unique_fields:

692 # Primary key is allowed in unique_fields.

693 unique_fields = [

694 self.model._meta.get_field(opts.pk.name if name == "pk" else name)

695 for name in unique_fields

696 ]

697 if update_fields:

698 update_fields = [self.model._meta.get_field(name) for name in update_fields]

699 on_conflict = self._check_bulk_create_options(

700 ignore_conflicts,

701 update_conflicts,

702 update_fields,

703 unique_fields,

704 )

705 self._for_write = True

706 fields = opts.concrete_fields

707 objs = list(objs)

708 self._prepare_for_bulk_create(objs)

709 with transaction.atomic(using=self.db, savepoint=False):

710 objs_with_pk, objs_without_pk = partition(lambda o: o.pk is None, objs)

711 if objs_with_pk:

712 returned_columns = self._batched_insert(

713 objs_with_pk,

714 fields,

715 batch_size,

716 on_conflict=on_conflict,

717 update_fields=update_fields,

718 unique_fields=unique_fields,

719 )

720 for obj_with_pk, results in zip(objs_with_pk, returned_columns):

721 for result, field in zip(results, opts.db_returning_fields):

722 if field != opts.pk:

723 setattr(obj_with_pk, field.attname, result)

724 for obj_with_pk in objs_with_pk:

725 obj_with_pk._state.adding = False

726 obj_with_pk._state.db = self.db

727 if objs_without_pk:

728 fields = [f for f in fields if not isinstance(f, AutoField)]

729 returned_columns = self._batched_insert(

730 objs_without_pk,

731 fields,

732 batch_size,

733 on_conflict=on_conflict,

734 update_fields=update_fields,

735 unique_fields=unique_fields,

736 )

737 connection = connections[self.db]

738 if (

739 connection.features.can_return_rows_from_bulk_insert

740 and on_conflict is None

741 ):

742 assert len(returned_columns) == len(objs_without_pk)

743 for obj_without_pk, results in zip(objs_without_pk, returned_columns):

744 for result, field in zip(results, opts.db_returning_fields):

745 setattr(obj_without_pk, field.attname, result)

746 obj_without_pk._state.adding = False

747 obj_without_pk._state.db = self.db

748

749 return objs

750

751 def bulk_update(self, objs, fields, batch_size=None):

752 """

753 Update the given fields in each of the given objects in the database.

754 """

755 if batch_size is not None and batch_size <= 0:

756 raise ValueError("Batch size must be a positive integer.")

757 if not fields:

758 raise ValueError("Field names must be given to bulk_update().")

759 objs = tuple(objs)

760 if any(obj.pk is None for obj in objs):

761 raise ValueError("All bulk_update() objects must have a primary key set.")

762 fields = [self.model._meta.get_field(name) for name in fields]

763 if any(not f.concrete or f.many_to_many for f in fields):

764 raise ValueError("bulk_update() can only be used with concrete fields.")

765 if any(f.primary_key for f in fields):

766 raise ValueError("bulk_update() cannot be used with primary key fields.")

767 if not objs:

768 return 0

769 for obj in objs:

770 obj._prepare_related_fields_for_save(

771 operation_name="bulk_update", fields=fields

772 )

773 # PK is used twice in the resulting update query, once in the filter

774 # and once in the WHEN. Each field will also have one CAST.

775 self._for_write = True

776 connection = connections[self.db]

777 max_batch_size = connection.ops.bulk_batch_size(["pk", "pk"] + fields, objs)

778 batch_size = min(batch_size, max_batch_size) if batch_size else max_batch_size

779 requires_casting = connection.features.requires_casted_case_in_updates

780 batches = (objs[i : i + batch_size] for i in range(0, len(objs), batch_size))

781 updates = []

782 for batch_objs in batches:

783 update_kwargs = {}

784 for field in fields:

785 when_statements = []

786 for obj in batch_objs:

787 attr = getattr(obj, field.attname)

788 if not hasattr(attr, "resolve_expression"):

789 attr = Value(attr, output_field=field)

790 when_statements.append(When(pk=obj.pk, then=attr))

791 case_statement = Case(*when_statements, output_field=field)

792 if requires_casting:

793 case_statement = Cast(case_statement, output_field=field)

794 update_kwargs[field.attname] = case_statement

795 updates.append(([obj.pk for obj in batch_objs], update_kwargs))

796 rows_updated = 0

797 queryset = self.using(self.db)

798 with transaction.atomic(using=self.db, savepoint=False):

799 for pks, update_kwargs in updates:

800 rows_updated += queryset.filter(pk__in=pks).update(**update_kwargs)

801 return rows_updated

802

803 bulk_update.alters_data = True

804

805 def get_or_create(self, defaults=None, **kwargs):

806 """

807 Look up an object with the given kwargs, creating one if necessary.

808 Return a tuple of (object, created), where created is a boolean

809 specifying whether an object was created.

810 """

811 # The get() needs to be targeted at the write database in order

812 # to avoid potential transaction consistency problems.

813 self._for_write = True

814 try:

815 return self.get(**kwargs), False

816 except self.model.DoesNotExist:

817 params = self._extract_model_params(defaults, **kwargs)

818 # Try to create an object using passed params.

819 try:

820 with transaction.atomic(using=self.db):

821 params = dict(resolve_callables(params))

822 return self.create(**params), True

823 except (IntegrityError, ValidationError):

824 # Since create() also validates by default,

825 # we can get any kind of ValidationError here,

826 # or it can flow through and get an IntegrityError from the database.

827 # The main thing we're concerned about is uniqueness failures,

828 # but ValidationError could include other things too.

829 # In all cases though it should be fine to try the get() again

830 # and return an existing object.

831 try:

832 return self.get(**kwargs), False

833 except self.model.DoesNotExist:

834 pass

835 raise

836

837 def update_or_create(self, defaults=None, create_defaults=None, **kwargs):

838 """

839 Look up an object with the given kwargs, updating one with defaults

840 if it exists, otherwise create a new one. Optionally, an object can

841 be created with different values than defaults by using

842 create_defaults.

843 Return a tuple (object, created), where created is a boolean

844 specifying whether an object was created.

845 """

846 if create_defaults is None:

847 update_defaults = create_defaults = defaults or {}

848 else:

849 update_defaults = defaults or {}

850 self._for_write = True

851 with transaction.atomic(using=self.db):

852 # Lock the row so that a concurrent update is blocked until

853 # update_or_create() has performed its save.

854 obj, created = self.select_for_update().get_or_create(

855 create_defaults, **kwargs

856 )

857 if created:

858 return obj, created

859 for k, v in resolve_callables(update_defaults):

860 setattr(obj, k, v)

861

862 update_fields = set(update_defaults)

863 concrete_field_names = self.model._meta._non_pk_concrete_field_names

864 # update_fields does not support non-concrete fields.

865 if concrete_field_names.issuperset(update_fields):

866 # Add fields which are set on pre_save(), e.g. auto_now fields.

867 # This is to maintain backward compatibility as these fields

868 # are not updated unless explicitly specified in the

869 # update_fields list.

870 for field in self.model._meta.local_concrete_fields:

871 if not (

872 field.primary_key or field.__class__.pre_save is Field.pre_save

873 ):

874 update_fields.add(field.name)

875 if field.name != field.attname:

876 update_fields.add(field.attname)

877 obj.save(using=self.db, update_fields=update_fields)

878 else:

879 obj.save(using=self.db)

880 return obj, False

881

882 def _extract_model_params(self, defaults, **kwargs):

883 """

884 Prepare `params` for creating a model instance based on the given

885 kwargs; for use by get_or_create().

886 """

887 defaults = defaults or {}

888 params = {k: v for k, v in kwargs.items() if LOOKUP_SEP not in k}

889 params.update(defaults)

890 property_names = self.model._meta._property_names

891 invalid_params = []

892 for param in params:

893 try:

894 self.model._meta.get_field(param)

895 except exceptions.FieldDoesNotExist:

896 # It's okay to use a model's property if it has a setter.

897 if not (param in property_names and getattr(self.model, param).fset):

898 invalid_params.append(param)

899 if invalid_params:

900 raise exceptions.FieldError(

901 "Invalid field name(s) for model {}: '{}'.".format(

902 self.model._meta.object_name,

903 "', '".join(sorted(invalid_params)),

904 )

905 )

906 return params

907

908 def _earliest(self, *fields):

909 """

910 Return the earliest object according to fields (if given) or by the

911 model's Meta.get_latest_by.

912 """

913 if fields:

914 order_by = fields

915 else:

916 order_by = getattr(self.model._meta, "get_latest_by")

917 if order_by and not isinstance(order_by, tuple | list):

918 order_by = (order_by,)

919 if order_by is None:

920 raise ValueError(

921 "earliest() and latest() require either fields as positional "

922 "arguments or 'get_latest_by' in the model's Meta."

923 )

924 obj = self._chain()

925 obj.query.set_limits(high=1)

926 obj.query.clear_ordering(force=True)

927 obj.query.add_ordering(*order_by)

928 return obj.get()

929

930 def earliest(self, *fields):

931 if self.query.is_sliced:

932 raise TypeError("Cannot change a query once a slice has been taken.")

933 return self._earliest(*fields)

934

935 def latest(self, *fields):

936 """

937 Return the latest object according to fields (if given) or by the

938 model's Meta.get_latest_by.

939 """

940 if self.query.is_sliced:

941 raise TypeError("Cannot change a query once a slice has been taken.")

942 return self.reverse()._earliest(*fields)

943

944 def first(self):

945 """Return the first object of a query or None if no match is found."""

946 if self.ordered:

947 queryset = self

948 else:

949 self._check_ordering_first_last_queryset_aggregation(method="first")

950 queryset = self.order_by("pk")

951 for obj in queryset[:1]:

952 return obj

953

954 def last(self):

955 """Return the last object of a query or None if no match is found."""

956 if self.ordered:

957 queryset = self.reverse()

958 else:

959 self._check_ordering_first_last_queryset_aggregation(method="last")

960 queryset = self.order_by("-pk")

961 for obj in queryset[:1]:

962 return obj

963

964 def in_bulk(self, id_list=None, *, field_name="pk"):

965 """

966 Return a dictionary mapping each of the given IDs to the object with

967 that ID. If `id_list` isn't provided, evaluate the entire QuerySet.

968 """

969 if self.query.is_sliced:

970 raise TypeError("Cannot use 'limit' or 'offset' with in_bulk().")

971 opts = self.model._meta

972 unique_fields = [

973 constraint.fields[0]

974 for constraint in opts.total_unique_constraints

975 if len(constraint.fields) == 1

976 ]

977 if (

978 field_name != "pk"

979 and not opts.get_field(field_name).unique

980 and field_name not in unique_fields

981 and self.query.distinct_fields != (field_name,)

982 ):

983 raise ValueError(

984 f"in_bulk()'s field_name must be a unique field but {field_name! r} isn't."

985 )

986 if id_list is not None:

987 if not id_list:

988 return {}

989 filter_key = f"{field_name}__in"

990 batch_size = connections[self.db].features.max_query_params

991 id_list = tuple(id_list)

992 # If the database has a limit on the number of query parameters

993 # (e.g. SQLite), retrieve objects in batches if necessary.

994 if batch_size and batch_size < len(id_list):

995 qs = ()

996 for offset in range(0, len(id_list), batch_size):

997 batch = id_list[offset : offset + batch_size]

998 qs += tuple(self.filter(**{filter_key: batch}))

999 else:

1000 qs = self.filter(**{filter_key: id_list})

1001 else:

1002 qs = self._chain()

1003 return {getattr(obj, field_name): obj for obj in qs}

1004

1005 def delete(self):

1006 """Delete the records in the current QuerySet."""

1007 self._not_support_combined_queries("delete")

1008 if self.query.is_sliced:

1009 raise TypeError("Cannot use 'limit' or 'offset' with delete().")

1010 if self.query.distinct or self.query.distinct_fields:

1011 raise TypeError("Cannot call delete() after .distinct().")

1012 if self._fields is not None:

1013 raise TypeError("Cannot call delete() after .values() or .values_list()")

1014

1015 del_query = self._chain()

1016

1017 # The delete is actually 2 queries - one to find related objects,

1018 # and one to delete. Make sure that the discovery of related

1019 # objects is performed on the same database as the deletion.

1020 del_query._for_write = True

1021

1022 # Disable non-supported fields.

1023 del_query.query.select_for_update = False

1024 del_query.query.select_related = False

1025 del_query.query.clear_ordering(force=True)

1026

1027 from plain.models.deletion import Collector

1028

1029 collector = Collector(using=del_query.db, origin=self)

1030 collector.collect(del_query)

1031 deleted, _rows_count = collector.delete()

1032

1033 # Clear the result cache, in case this QuerySet gets reused.

1034 self._result_cache = None

1035 return deleted, _rows_count

1036

1037 delete.alters_data = True

1038 delete.queryset_only = True

1039

1040 def _raw_delete(self, using):

1041 """

1042 Delete objects found from the given queryset in single direct SQL

1043 query. No signals are sent and there is no protection for cascades.

1044 """

1045 query = self.query.clone()

1046 query.__class__ = sql.DeleteQuery

1047 cursor = query.get_compiler(using).execute_sql(CURSOR)

1048 if cursor:

1049 with cursor:

1050 return cursor.rowcount

1051 return 0

1052

1053 _raw_delete.alters_data = True

1054

1055 def update(self, **kwargs):

1056 """

1057 Update all elements in the current QuerySet, setting all the given

1058 fields to the appropriate values.

1059 """

1060 self._not_support_combined_queries("update")

1061 if self.query.is_sliced:

1062 raise TypeError("Cannot update a query once a slice has been taken.")

1063 self._for_write = True

1064 query = self.query.chain(sql.UpdateQuery)

1065 query.add_update_values(kwargs)

1066

1067 # Inline annotations in order_by(), if possible.

1068 new_order_by = []

1069 for col in query.order_by:

1070 alias = col

1071 descending = False

1072 if isinstance(alias, str) and alias.startswith("-"):

1073 alias = alias.removeprefix("-")

1074 descending = True

1075 if annotation := query.annotations.get(alias):

1076 if getattr(annotation, "contains_aggregate", False):

1077 raise exceptions.FieldError(

1078 f"Cannot update when ordering by an aggregate: {annotation}"

1079 )

1080 if descending:

1081 annotation = annotation.desc()

1082 new_order_by.append(annotation)

1083 else:

1084 new_order_by.append(col)

1085 query.order_by = tuple(new_order_by)

1086

1087 # Clear any annotations so that they won't be present in subqueries.

1088 query.annotations = {}

1089 with transaction.mark_for_rollback_on_error(using=self.db):

1090 rows = query.get_compiler(self.db).execute_sql(CURSOR)

1091 self._result_cache = None

1092 return rows

1093

1094 update.alters_data = True

1095

1096 def _update(self, values):

1097 """

1098 A version of update() that accepts field objects instead of field names.

1099 Used primarily for model saving and not intended for use by general

1100 code (it requires too much poking around at model internals to be

1101 useful at that level).

1102 """

1103 if self.query.is_sliced:

1104 raise TypeError("Cannot update a query once a slice has been taken.")

1105 query = self.query.chain(sql.UpdateQuery)

1106 query.add_update_fields(values)

1107 # Clear any annotations so that they won't be present in subqueries.

1108 query.annotations = {}

1109 self._result_cache = None

1110 return query.get_compiler(self.db).execute_sql(CURSOR)

1111

1112 _update.alters_data = True

1113 _update.queryset_only = False

1114

1115 def exists(self):

1116 """

1117 Return True if the QuerySet would have any results, False otherwise.

1118 """

1119 if self._result_cache is None:

1120 return self.query.has_results(using=self.db)

1121 return bool(self._result_cache)

1122

1123 def contains(self, obj):

1124 """

1125 Return True if the QuerySet contains the provided obj,

1126 False otherwise.

1127 """

1128 self._not_support_combined_queries("contains")

1129 if self._fields is not None:

1130 raise TypeError(

1131 "Cannot call QuerySet.contains() after .values() or .values_list()."

1132 )

1133 try:

1134 if obj._meta.concrete_model != self.model._meta.concrete_model:

1135 return False

1136 except AttributeError:

1137 raise TypeError("'obj' must be a model instance.")

1138 if obj.pk is None:

1139 raise ValueError("QuerySet.contains() cannot be used on unsaved objects.")

1140 if self._result_cache is not None:

1141 return obj in self._result_cache

1142 return self.filter(pk=obj.pk).exists()

1143

1144 def _prefetch_related_objects(self):

1145 # This method can only be called once the result cache has been filled.

1146 prefetch_related_objects(self._result_cache, *self._prefetch_related_lookups)

1147 self._prefetch_done = True

1148

1149 def explain(self, *, format=None, **options):

1150 """

1151 Runs an EXPLAIN on the SQL query this QuerySet would perform, and

1152 returns the results.

1153 """

1154 return self.query.explain(using=self.db, format=format, **options)

1155

1156 ##################################################

1157 # PUBLIC METHODS THAT RETURN A QUERYSET SUBCLASS #

1158 ##################################################

1159

1160 def raw(self, raw_query, params=(), translations=None, using=None):

1161 if using is None:

1162 using = self.db

1163 qs = RawQuerySet(

1164 raw_query,

1165 model=self.model,

1166 params=params,

1167 translations=translations,

1168 using=using,

1169 )

1170 qs._prefetch_related_lookups = self._prefetch_related_lookups[:]

1171 return qs

1172

1173 def _values(self, *fields, **expressions):

1174 clone = self._chain()

1175 if expressions:

1176 clone = clone.annotate(**expressions)

1177 clone._fields = fields

1178 clone.query.set_values(fields)

1179 return clone

1180

1181 def values(self, *fields, **expressions):

1182 fields += tuple(expressions)

1183 clone = self._values(*fields, **expressions)

1184 clone._iterable_class = ValuesIterable

1185 return clone

1186

1187 def values_list(self, *fields, flat=False, named=False):

1188 if flat and named:

1189 raise TypeError("'flat' and 'named' can't be used together.")

1190 if flat and len(fields) > 1:

1191 raise TypeError(

1192 "'flat' is not valid when values_list is called with more than one "

1193 "field."

1194 )

1195

1196 field_names = {f for f in fields if not hasattr(f, "resolve_expression")}

1197 _fields = []

1198 expressions = {}

1199 counter = 1

1200 for field in fields:

1201 if hasattr(field, "resolve_expression"):

1202 field_id_prefix = getattr(

1203 field, "default_alias", field.__class__.__name__.lower()

1204 )

1205 while True:

1206 field_id = field_id_prefix + str(counter)

1207 counter += 1

1208 if field_id not in field_names:

1209 break

1210 expressions[field_id] = field

1211 _fields.append(field_id)

1212 else:

1213 _fields.append(field)

1214

1215 clone = self._values(*_fields, **expressions)

1216 clone._iterable_class = (

1217 NamedValuesListIterable

1218 if named

1219 else FlatValuesListIterable

1220 if flat

1221 else ValuesListIterable

1222 )

1223 return clone

1224

1225 def dates(self, field_name, kind, order="ASC"):

1226 """

1227 Return a list of date objects representing all available dates for

1228 the given field_name, scoped to 'kind'.

1229 """

1230 if kind not in ("year", "month", "week", "day"):

1231 raise ValueError("'kind' must be one of 'year', 'month', 'week', or 'day'.")

1232 if order not in ("ASC", "DESC"):

1233 raise ValueError("'order' must be either 'ASC' or 'DESC'.")

1234 return (

1235 self.annotate(

1236 datefield=Trunc(field_name, kind, output_field=DateField()),

1237 plain_field=F(field_name),

1238 )

1239 .values_list("datefield", flat=True)

1240 .distinct()

1241 .filter(plain_field__isnull=False)

1242 .order_by(("-" if order == "DESC" else "") + "datefield")

1243 )

1244

1245 def datetimes(self, field_name, kind, order="ASC", tzinfo=None):

1246 """

1247 Return a list of datetime objects representing all available

1248 datetimes for the given field_name, scoped to 'kind'.

1249 """

1250 if kind not in ("year", "month", "week", "day", "hour", "minute", "second"):

1251 raise ValueError(

1252 "'kind' must be one of 'year', 'month', 'week', 'day', "

1253 "'hour', 'minute', or 'second'."

1254 )

1255 if order not in ("ASC", "DESC"):

1256 raise ValueError("'order' must be either 'ASC' or 'DESC'.")

1257

1258 if tzinfo is None:

1259 tzinfo = timezone.get_current_timezone()

1260

1261 return (

1262 self.annotate(

1263 datetimefield=Trunc(

1264 field_name,

1265 kind,

1266 output_field=DateTimeField(),

1267 tzinfo=tzinfo,

1268 ),

1269 plain_field=F(field_name),

1270 )

1271 .values_list("datetimefield", flat=True)

1272 .distinct()

1273 .filter(plain_field__isnull=False)

1274 .order_by(("-" if order == "DESC" else "") + "datetimefield")

1275 )

1276

1277 def none(self):

1278 """Return an empty QuerySet."""

1279 clone = self._chain()

1280 clone.query.set_empty()

1281 return clone

1282

1283 ##################################################################

1284 # PUBLIC METHODS THAT ALTER ATTRIBUTES AND RETURN A NEW QUERYSET #

1285 ##################################################################

1286

1287 def all(self):

1288 """

1289 Return a new QuerySet that is a copy of the current one. This allows a

1290 QuerySet to proxy for a model manager in some cases.

1291 """

1292 return self._chain()

1293

1294 def filter(self, *args, **kwargs):

1295 """

1296 Return a new QuerySet instance with the args ANDed to the existing

1297 set.

1298 """

1299 self._not_support_combined_queries("filter")

1300 return self._filter_or_exclude(False, args, kwargs)

1301

1302 def exclude(self, *args, **kwargs):

1303 """

1304 Return a new QuerySet instance with NOT (args) ANDed to the existing

1305 set.

1306 """

1307 self._not_support_combined_queries("exclude")

1308 return self._filter_or_exclude(True, args, kwargs)

1309

1310 def _filter_or_exclude(self, negate, args, kwargs):

1311 if (args or kwargs) and self.query.is_sliced:

1312 raise TypeError("Cannot filter a query once a slice has been taken.")

1313 clone = self._chain()

1314 if self._defer_next_filter:

1315 self._defer_next_filter = False

1316 clone._deferred_filter = negate, args, kwargs

1317 else:

1318 clone._filter_or_exclude_inplace(negate, args, kwargs)

1319 return clone

1320

1321 def _filter_or_exclude_inplace(self, negate, args, kwargs):

1322 if negate:

1323 self._query.add_q(~Q(*args, **kwargs))

1324 else:

1325 self._query.add_q(Q(*args, **kwargs))

1326

1327 def complex_filter(self, filter_obj):

1328 """

1329 Return a new QuerySet instance with filter_obj added to the filters.

1330

1331 filter_obj can be a Q object or a dictionary of keyword lookup

1332 arguments.

1333

1334 This exists to support framework features such as 'limit_choices_to',

1335 and usually it will be more natural to use other methods.

1336 """

1337 if isinstance(filter_obj, Q):

1338 clone = self._chain()

1339 clone.query.add_q(filter_obj)

1340 return clone

1341 else:

1342 return self._filter_or_exclude(False, args=(), kwargs=filter_obj)

1343

1344 def _combinator_query(self, combinator, *other_qs, all=False):

1345 # Clone the query to inherit the select list and everything

1346 clone = self._chain()

1347 # Clear limits and ordering so they can be reapplied

1348 clone.query.clear_ordering(force=True)

1349 clone.query.clear_limits()

1350 clone.query.combined_queries = (self.query,) + tuple(

1351 qs.query for qs in other_qs

1352 )

1353 clone.query.combinator = combinator

1354 clone.query.combinator_all = all

1355 return clone

1356

1357 def union(self, *other_qs, all=False):

1358 # If the query is an EmptyQuerySet, combine all nonempty querysets.

1359 if isinstance(self, EmptyQuerySet):

1360 qs = [q for q in other_qs if not isinstance(q, EmptyQuerySet)]

1361 if not qs:

1362 return self

1363 if len(qs) == 1:

1364 return qs[0]

1365 return qs[0]._combinator_query("union", *qs[1:], all=all)

1366 return self._combinator_query("union", *other_qs, all=all)

1367

1368 def intersection(self, *other_qs):

1369 # If any query is an EmptyQuerySet, return it.

1370 if isinstance(self, EmptyQuerySet):

1371 return self

1372 for other in other_qs:

1373 if isinstance(other, EmptyQuerySet):

1374 return other

1375 return self._combinator_query("intersection", *other_qs)

1376

1377 def difference(self, *other_qs):

1378 # If the query is an EmptyQuerySet, return it.

1379 if isinstance(self, EmptyQuerySet):

1380 return self

1381 return self._combinator_query("difference", *other_qs)

1382

1383 def select_for_update(self, nowait=False, skip_locked=False, of=(), no_key=False):

1384 """

1385 Return a new QuerySet instance that will select objects with a

1386 FOR UPDATE lock.

1387 """

1388 if nowait and skip_locked:

1389 raise ValueError("The nowait option cannot be used with skip_locked.")

1390 obj = self._chain()

1391 obj._for_write = True

1392 obj.query.select_for_update = True

1393 obj.query.select_for_update_nowait = nowait

1394 obj.query.select_for_update_skip_locked = skip_locked

1395 obj.query.select_for_update_of = of

1396 obj.query.select_for_no_key_update = no_key

1397 return obj

1398

1399 def select_related(self, *fields):

1400 """

1401 Return a new QuerySet instance that will select related objects.

1402

1403 If fields are specified, they must be ForeignKey fields and only those

1404 related objects are included in the selection.

1405

1406 If select_related(None) is called, clear the list.

1407 """

1408 self._not_support_combined_queries("select_related")

1409 if self._fields is not None:

1410 raise TypeError(

1411 "Cannot call select_related() after .values() or .values_list()"

1412 )

1413

1414 obj = self._chain()

1415 if fields == (None,):

1416 obj.query.select_related = False

1417 elif fields:

1418 obj.query.add_select_related(fields)

1419 else:

1420 obj.query.select_related = True

1421 return obj

1422

1423 def prefetch_related(self, *lookups):

1424 """

1425 Return a new QuerySet instance that will prefetch the specified

1426 Many-To-One and Many-To-Many related objects when the QuerySet is

1427 evaluated.

1428

1429 When prefetch_related() is called more than once, append to the list of

1430 prefetch lookups. If prefetch_related(None) is called, clear the list.

1431 """

1432 self._not_support_combined_queries("prefetch_related")

1433 clone = self._chain()

1434 if lookups == (None,):

1435 clone._prefetch_related_lookups = ()

1436 else:

1437 for lookup in lookups:

1438 if isinstance(lookup, Prefetch):

1439 lookup = lookup.prefetch_to

1440 lookup = lookup.split(LOOKUP_SEP, 1)[0]

1441 if lookup in self.query._filtered_relations:

1442 raise ValueError(

1443 "prefetch_related() is not supported with FilteredRelation."

1444 )

1445 clone._prefetch_related_lookups = clone._prefetch_related_lookups + lookups

1446 return clone

1447

1448 def annotate(self, *args, **kwargs):

1449 """

1450 Return a query set in which the returned objects have been annotated

1451 with extra data or aggregations.

1452 """

1453 self._not_support_combined_queries("annotate")

1454 return self._annotate(args, kwargs, select=True)

1455

1456 def alias(self, *args, **kwargs):

1457 """

1458 Return a query set with added aliases for extra data or aggregations.

1459 """

1460 self._not_support_combined_queries("alias")

1461 return self._annotate(args, kwargs, select=False)

1462

1463 def _annotate(self, args, kwargs, select=True):

1464 self._validate_values_are_expressions(

1465 args + tuple(kwargs.values()), method_name="annotate"

1466 )

1467 annotations = {}

1468 for arg in args:

1469 # The default_alias property may raise a TypeError.

1470 try:

1471 if arg.default_alias in kwargs:

1472 raise ValueError(

1473 f"The named annotation '{arg.default_alias}' conflicts with the "

1474 "default name for another annotation."

1475 )

1476 except TypeError:

1477 raise TypeError("Complex annotations require an alias")

1478 annotations[arg.default_alias] = arg

1479 annotations.update(kwargs)

1480

1481 clone = self._chain()

1482 names = self._fields

1483 if names is None:

1484 names = set(

1485 chain.from_iterable(

1486 (field.name, field.attname)

1487 if hasattr(field, "attname")

1488 else (field.name,)

1489 for field in self.model._meta.get_fields()

1490 )

1491 )

1492

1493 for alias, annotation in annotations.items():

1494 if alias in names:

1495 raise ValueError(

1496 f"The annotation '{alias}' conflicts with a field on " "the model."

1497 )

1498 if isinstance(annotation, FilteredRelation):

1499 clone.query.add_filtered_relation(annotation, alias)

1500 else:

1501 clone.query.add_annotation(

1502 annotation,

1503 alias,

1504 select=select,

1505 )

1506 for alias, annotation in clone.query.annotations.items():

1507 if alias in annotations and annotation.contains_aggregate:

1508 if clone._fields is None:

1509 clone.query.group_by = True

1510 else:

1511 clone.query.set_group_by()

1512 break

1513

1514 return clone

1515

1516 def order_by(self, *field_names):

1517 """Return a new QuerySet instance with the ordering changed."""

1518 if self.query.is_sliced:

1519 raise TypeError("Cannot reorder a query once a slice has been taken.")

1520 obj = self._chain()

1521 obj.query.clear_ordering(force=True, clear_default=False)

1522 obj.query.add_ordering(*field_names)

1523 return obj

1524

1525 def distinct(self, *field_names):

1526 """

1527 Return a new QuerySet instance that will select only distinct results.

1528 """

1529 self._not_support_combined_queries("distinct")

1530 if self.query.is_sliced:

1531 raise TypeError(

1532 "Cannot create distinct fields once a slice has been taken."

1533 )

1534 obj = self._chain()

1535 obj.query.add_distinct_fields(*field_names)

1536 return obj

1537

1538 def extra(

1539 self,

1540 select=None,

1541 where=None,

1542 params=None,

1543 tables=None,

1544 order_by=None,

1545 select_params=None,

1546 ):

1547 """Add extra SQL fragments to the query."""

1548 self._not_support_combined_queries("extra")

1549 if self.query.is_sliced:

1550 raise TypeError("Cannot change a query once a slice has been taken.")

1551 clone = self._chain()

1552 clone.query.add_extra(select, select_params, where, params, tables, order_by)

1553 return clone

1554

1555 def reverse(self):

1556 """Reverse the ordering of the QuerySet."""

1557 if self.query.is_sliced:

1558 raise TypeError("Cannot reverse a query once a slice has been taken.")

1559 clone = self._chain()

1560 clone.query.standard_ordering = not clone.query.standard_ordering

1561 return clone

1562

1563 def defer(self, *fields):

1564 """

1565 Defer the loading of data for certain fields until they are accessed.

1566 Add the set of deferred fields to any existing set of deferred fields.

1567 The only exception to this is if None is passed in as the only

1568 parameter, in which case removal all deferrals.

1569 """

1570 self._not_support_combined_queries("defer")

1571 if self._fields is not None:

1572 raise TypeError("Cannot call defer() after .values() or .values_list()")

1573 clone = self._chain()

1574 if fields == (None,):

1575 clone.query.clear_deferred_loading()

1576 else:

1577 clone.query.add_deferred_loading(fields)

1578 return clone

1579

1580 def only(self, *fields):

1581 """

1582 Essentially, the opposite of defer(). Only the fields passed into this

1583 method and that are not already specified as deferred are loaded

1584 immediately when the queryset is evaluated.

1585 """

1586 self._not_support_combined_queries("only")

1587 if self._fields is not None:

1588 raise TypeError("Cannot call only() after .values() or .values_list()")

1589 if fields == (None,):

1590 # Can only pass None to defer(), not only(), as the rest option.

1591 # That won't stop people trying to do this, so let's be explicit.

1592 raise TypeError("Cannot pass None as an argument to only().")

1593 for field in fields:

1594 field = field.split(LOOKUP_SEP, 1)[0]

1595 if field in self.query._filtered_relations:

1596 raise ValueError("only() is not supported with FilteredRelation.")

1597 clone = self._chain()

1598 clone.query.add_immediate_loading(fields)

1599 return clone

1600

1601 def using(self, alias):

1602 """Select which database this QuerySet should execute against."""

1603 clone = self._chain()

1604 clone._db = alias

1605 return clone

1606

1607 ###################################

1608 # PUBLIC INTROSPECTION ATTRIBUTES #

1609 ###################################

1610

1611 @property

1612 def ordered(self):

1613 """

1614 Return True if the QuerySet is ordered -- i.e. has an order_by()

1615 clause or a default ordering on the model (or is empty).

1616 """

1617 if isinstance(self, EmptyQuerySet):

1618 return True

1619 if self.query.extra_order_by or self.query.order_by:

1620 return True

1621 elif (

1622 self.query.default_ordering

1623 and self.query.get_meta().ordering

1624 and

1625 # A default ordering doesn't affect GROUP BY queries.

1626 not self.query.group_by

1627 ):

1628 return True

1629 else:

1630 return False

1631

1632 @property

1633 def db(self):

1634 """Return the database used if this query is executed now."""

1635 if self._for_write:

1636 return self._db or router.db_for_write(self.model, **self._hints)

1637 return self._db or router.db_for_read(self.model, **self._hints)

1638

1639 ###################

1640 # PRIVATE METHODS #

1641 ###################

1642

1643 def _insert(

1644 self,

1645 objs,

1646 fields,

1647 returning_fields=None,

1648 raw=False,

1649 using=None,

1650 on_conflict=None,

1651 update_fields=None,

1652 unique_fields=None,

1653 ):

1654 """

1655 Insert a new record for the given model. This provides an interface to

1656 the InsertQuery class and is how Model.save() is implemented.

1657 """

1658 self._for_write = True

1659 if using is None:

1660 using = self.db

1661 query = sql.InsertQuery(

1662 self.model,

1663 on_conflict=on_conflict,

1664 update_fields=update_fields,

1665 unique_fields=unique_fields,

1666 )

1667 query.insert_values(fields, objs, raw=raw)

1668 return query.get_compiler(using=using).execute_sql(returning_fields)

1669

1670 _insert.alters_data = True

1671 _insert.queryset_only = False

1672

1673 def _batched_insert(

1674 self,

1675 objs,

1676 fields,

1677 batch_size,

1678 on_conflict=None,

1679 update_fields=None,

1680 unique_fields=None,

1681 ):

1682 """

1683 Helper method for bulk_create() to insert objs one batch at a time.

1684 """

1685 connection = connections[self.db]

1686 ops = connection.ops

1687 max_batch_size = max(ops.bulk_batch_size(fields, objs), 1)

1688 batch_size = min(batch_size, max_batch_size) if batch_size else max_batch_size

1689 inserted_rows = []

1690 bulk_return = connection.features.can_return_rows_from_bulk_insert

1691 for item in [objs[i : i + batch_size] for i in range(0, len(objs), batch_size)]:

1692 if bulk_return and on_conflict is None:

1693 inserted_rows.extend(

1694 self._insert(

1695 item,

1696 fields=fields,

1697 using=self.db,

1698 returning_fields=self.model._meta.db_returning_fields,

1699 )

1700 )

1701 else:

1702 self._insert(

1703 item,

1704 fields=fields,

1705 using=self.db,

1706 on_conflict=on_conflict,

1707 update_fields=update_fields,

1708 unique_fields=unique_fields,

1709 )

1710 return inserted_rows

1711

1712 def _chain(self):

1713 """

1714 Return a copy of the current QuerySet that's ready for another

1715 operation.

1716 """

1717 obj = self._clone()

1718 if obj._sticky_filter:

1719 obj.query.filter_is_sticky = True

1720 obj._sticky_filter = False

1721 return obj

1722

1723 def _clone(self):

1724 """

1725 Return a copy of the current QuerySet. A lightweight alternative

1726 to deepcopy().

1727 """

1728 c = self.__class__(

1729 model=self.model,

1730 query=self.query.chain(),

1731 using=self._db,

1732 hints=self._hints,

1733 )

1734 c._sticky_filter = self._sticky_filter

1735 c._for_write = self._for_write

1736 c._prefetch_related_lookups = self._prefetch_related_lookups[:]

1737 c._known_related_objects = self._known_related_objects

1738 c._iterable_class = self._iterable_class

1739 c._fields = self._fields

1740 return c

1741

1742 def _fetch_all(self):

1743 if self._result_cache is None:

1744 self._result_cache = list(self._iterable_class(self))

1745 if self._prefetch_related_lookups and not self._prefetch_done:

1746 self._prefetch_related_objects()

1747

1748 def _next_is_sticky(self):

1749 """

1750 Indicate that the next filter call and the one following that should

1751 be treated as a single filter. This is only important when it comes to

1752 determining when to reuse tables for many-to-many filters. Required so

1753 that we can filter naturally on the results of related managers.

1754

1755 This doesn't return a clone of the current QuerySet (it returns

1756 "self"). The method is only used internally and should be immediately

1757 followed by a filter() that does create a clone.

1758 """

1759 self._sticky_filter = True

1760 return self

1761

1762 def _merge_sanity_check(self, other):

1763 """Check that two QuerySet classes may be merged."""

1764 if self._fields is not None and (

1765 set(self.query.values_select) != set(other.query.values_select)

1766 or set(self.query.extra_select) != set(other.query.extra_select)

1767 or set(self.query.annotation_select) != set(other.query.annotation_select)

1768 ):

1769 raise TypeError(

1770 f"Merging '{self.__class__.__name__}' classes must involve the same values in each case."

1771 )

1772

1773 def _merge_known_related_objects(self, other):

1774 """

1775 Keep track of all known related objects from either QuerySet instance.

1776 """

1777 for field, objects in other._known_related_objects.items():

1778 self._known_related_objects.setdefault(field, {}).update(objects)

1779

1780 def resolve_expression(self, *args, **kwargs):

1781 if self._fields and len(self._fields) > 1:

1782 # values() queryset can only be used as nested queries

1783 # if they are set up to select only a single field.

1784 raise TypeError("Cannot use multi-field values as a filter value.")

1785 query = self.query.resolve_expression(*args, **kwargs)

1786 query._db = self._db

1787 return query

1788

1789 resolve_expression.queryset_only = True

1790

1791 def _add_hints(self, **hints):

1792 """

1793 Update hinting information for use by routers. Add new key/values or

1794 overwrite existing key/values.

1795 """

1796 self._hints.update(hints)

1797

1798 def _has_filters(self):

1799 """

1800 Check if this QuerySet has any filtering going on. This isn't

1801 equivalent with checking if all objects are present in results, for

1802 example, qs[1:]._has_filters() -> False.

1803 """

1804 return self.query.has_filters()

1805

1806 @staticmethod

1807 def _validate_values_are_expressions(values, method_name):

1808 invalid_args = sorted(

1809 str(arg) for arg in values if not hasattr(arg, "resolve_expression")

1810 )

1811 if invalid_args:

1812 raise TypeError(

1813 "QuerySet.{}() received non-expression(s): {}.".format(

1814 method_name,

1815 ", ".join(invalid_args),

1816 )

1817 )

1818

1819 def _not_support_combined_queries(self, operation_name):

1820 if self.query.combinator:

1821 raise NotSupportedError(

1822 f"Calling QuerySet.{operation_name}() after {self.query.combinator}() is not supported."

1823 )

1824

1825 def _check_operator_queryset(self, other, operator_):

1826 if self.query.combinator or other.query.combinator:

1827 raise TypeError(f"Cannot use {operator_} operator with combined queryset.")

1828

1829 def _check_ordering_first_last_queryset_aggregation(self, method):

1830 if isinstance(self.query.group_by, tuple) and not any(

1831 col.output_field is self.model._meta.pk for col in self.query.group_by

1832 ):

1833 raise TypeError(

1834 f"Cannot use QuerySet.{method}() on an unordered queryset performing "

1835 f"aggregation. Add an ordering with order_by()."

1836 )

1837

1838

1839 class InstanceCheckMeta(type):

1840 def __instancecheck__(self, instance):

1841 return isinstance(instance, QuerySet) and instance.query.is_empty()

1842

1843

1844 class EmptyQuerySet(metaclass=InstanceCheckMeta):

1845 """

1846 Marker class to checking if a queryset is empty by .none():

1847 isinstance(qs.none(), EmptyQuerySet) -> True

1848 """

1849

1850 def __init__(self, *args, **kwargs):

1851 raise TypeError("EmptyQuerySet can't be instantiated")

1852

1853

1854 class RawQuerySet:

1855 """

1856 Provide an iterator which converts the results of raw SQL queries into

1857 annotated model instances.

1858 """

1859

1860 def __init__(

1861 self,

1862 raw_query,

1863 model=None,

1864 query=None,

1865 params=(),

1866 translations=None,

1867 using=None,

1868 hints=None,

1869 ):

1870 self.raw_query = raw_query

1871 self.model = model

1872 self._db = using

1873 self._hints = hints or {}

1874 self.query = query or sql.RawQuery(sql=raw_query, using=self.db, params=params)

1875 self.params = params

1876 self.translations = translations or {}

1877 self._result_cache = None

1878 self._prefetch_related_lookups = ()

1879 self._prefetch_done = False

1880

1881 def resolve_model_init_order(self):

1882 """Resolve the init field names and value positions."""

1883 converter = connections[self.db].introspection.identifier_converter

1884 model_init_fields = [

1885 f for f in self.model._meta.fields if converter(f.column) in self.columns

1886 ]

1887 annotation_fields = [

1888 (column, pos)

1889 for pos, column in enumerate(self.columns)

1890 if column not in self.model_fields

1891 ]

1892 model_init_order = [

1893 self.columns.index(converter(f.column)) for f in model_init_fields

1894 ]

1895 model_init_names = [f.attname for f in model_init_fields]

1896 return model_init_names, model_init_order, annotation_fields

1897

1898 def prefetch_related(self, *lookups):

1899 """Same as QuerySet.prefetch_related()"""

1900 clone = self._clone()

1901 if lookups == (None,):

1902 clone._prefetch_related_lookups = ()

1903 else:

1904 clone._prefetch_related_lookups = clone._prefetch_related_lookups + lookups

1905 return clone

1906

1907 def _prefetch_related_objects(self):

1908 prefetch_related_objects(self._result_cache, *self._prefetch_related_lookups)

1909 self._prefetch_done = True

1910

1911 def _clone(self):

1912 """Same as QuerySet._clone()"""

1913 c = self.__class__(

1914 self.raw_query,

1915 model=self.model,

1916 query=self.query,

1917 params=self.params,

1918 translations=self.translations,

1919 using=self._db,

1920 hints=self._hints,

1921 )

1922 c._prefetch_related_lookups = self._prefetch_related_lookups[:]

1923 return c

1924

1925 def _fetch_all(self):

1926 if self._result_cache is None:

1927 self._result_cache = list(self.iterator())

1928 if self._prefetch_related_lookups and not self._prefetch_done:

1929 self._prefetch_related_objects()

1930

1931 def __len__(self):

1932 self._fetch_all()

1933 return len(self._result_cache)

1934

1935 def __bool__(self):

1936 self._fetch_all()

1937 return bool(self._result_cache)

1938

1939 def __iter__(self):

1940 self._fetch_all()

1941 return iter(self._result_cache)

1942

1943 def iterator(self):

1944 yield from RawModelIterable(self)

1945

1946 def __repr__(self):

1947 return f"<{self.__class__.__name__}: {self.query}>"

1948

1949 def __getitem__(self, k):

1950 return list(self)[k]

1951

1952 @property

1953 def db(self):

1954 """Return the database used if this query is executed now."""

1955 return self._db or router.db_for_read(self.model, **self._hints)

1956

1957 def using(self, alias):

1958 """Select the database this RawQuerySet should execute against."""

1959 return RawQuerySet(

1960 self.raw_query,

1961 model=self.model,

1962 query=self.query.chain(using=alias),

1963 params=self.params,

1964 translations=self.translations,

1965 using=alias,

1966 )

1967

1968 @cached_property

1969 def columns(self):

1970 """

1971 A list of model field names in the order they'll appear in the

1972 query results.

1973 """

1974 columns = self.query.get_columns()

1975 # Adjust any column names which don't match field names

1976 for query_name, model_name in self.translations.items():

1977 # Ignore translations for nonexistent column names

1978 try:

1979 index = columns.index(query_name)

1980 except ValueError:

1981 pass

1982 else:

1983 columns[index] = model_name

1984 return columns

1985

1986 @cached_property

1987 def model_fields(self):

1988 """A dict mapping column names to model field names."""

1989 converter = connections[self.db].introspection.identifier_converter

1990 model_fields = {}

1991 for field in self.model._meta.fields:

1992 name, column = field.get_attname_column()

1993 model_fields[converter(column)] = field

1994 return model_fields

1995

1996

1997 class Prefetch:

1998 def __init__(self, lookup, queryset=None, to_attr=None):

1999 # `prefetch_through` is the path we traverse to perform the prefetch.

2000 self.prefetch_through = lookup

2001 # `prefetch_to` is the path to the attribute that stores the result.

2002 self.prefetch_to = lookup

2003 if queryset is not None and (

2004 isinstance(queryset, RawQuerySet)

2005 or (

2006 hasattr(queryset, "_iterable_class")

2007 and not issubclass(queryset._iterable_class, ModelIterable)

2008 )

2009 ):

2010 raise ValueError(

2011 "Prefetch querysets cannot use raw(), values(), and values_list()."

2012 )

2013 if to_attr:

2014 self.prefetch_to = LOOKUP_SEP.join(

2015 lookup.split(LOOKUP_SEP)[:-1] + [to_attr]

2016 )

2017

2018 self.queryset = queryset

2019 self.to_attr = to_attr

2020

2021 def __getstate__(self):

2022 obj_dict = self.__dict__.copy()

2023 if self.queryset is not None:

2024 queryset = self.queryset._chain()

2025 # Prevent the QuerySet from being evaluated

2026 queryset._result_cache = []

2027 queryset._prefetch_done = True

2028 obj_dict["queryset"] = queryset

2029 return obj_dict

2030

2031 def add_prefix(self, prefix):

2032 self.prefetch_through = prefix + LOOKUP_SEP + self.prefetch_through

2033 self.prefetch_to = prefix + LOOKUP_SEP + self.prefetch_to

2034

2035 def get_current_prefetch_to(self, level):

2036 return LOOKUP_SEP.join(self.prefetch_to.split(LOOKUP_SEP)[: level + 1])

2037

2038 def get_current_to_attr(self, level):

2039 parts = self.prefetch_to.split(LOOKUP_SEP)

2040 to_attr = parts[level]

2041 as_attr = self.to_attr and level == len(parts) - 1

2042 return to_attr, as_attr

2043

2044 def get_current_queryset(self, level):

2045 if self.get_current_prefetch_to(level) == self.prefetch_to:

2046 return self.queryset

2047 return None

2048

2049 def __eq__(self, other):

2050 if not isinstance(other, Prefetch):

2051 return NotImplemented

2052 return self.prefetch_to == other.prefetch_to

2053

2054 def __hash__(self):

2055 return hash((self.__class__, self.prefetch_to))

2056

2057

2058 def normalize_prefetch_lookups(lookups, prefix=None):

2059 """Normalize lookups into Prefetch objects."""

2060 ret = []

2061 for lookup in lookups:

2062 if not isinstance(lookup, Prefetch):

2063 lookup = Prefetch(lookup)

2064 if prefix:

2065 lookup.add_prefix(prefix)

2066 ret.append(lookup)

2067 return ret

2068

2069

2070 def prefetch_related_objects(model_instances, *related_lookups):

2071 """

2072 Populate prefetched object caches for a list of model instances based on

2073 the lookups/Prefetch instances given.

2074 """

2075 if not model_instances:

2076 return # nothing to do

2077

2078 # We need to be able to dynamically add to the list of prefetch_related

2079 # lookups that we look up (see below). So we need some book keeping to

2080 # ensure we don't do duplicate work.

2081 done_queries = {} # dictionary of things like 'foo__bar': [results]

2082

2083 auto_lookups = set() # we add to this as we go through.

2084 followed_descriptors = set() # recursion protection

2085

2086 all_lookups = normalize_prefetch_lookups(reversed(related_lookups))

2087 while all_lookups:

2088 lookup = all_lookups.pop()

2089 if lookup.prefetch_to in done_queries:

2090 if lookup.queryset is not None:

2091 raise ValueError(

2092 f"'{lookup.prefetch_to}' lookup was already seen with a different queryset. "

2093 "You may need to adjust the ordering of your lookups."

2094 )

2095

2096 continue

2097

2098 # Top level, the list of objects to decorate is the result cache

2099 # from the primary QuerySet. It won't be for deeper levels.

2100 obj_list = model_instances

2101

2102 through_attrs = lookup.prefetch_through.split(LOOKUP_SEP)

2103 for level, through_attr in enumerate(through_attrs):

2104 # Prepare main instances

2105 if not obj_list:

2106 break

2107

2108 prefetch_to = lookup.get_current_prefetch_to(level)

2109 if prefetch_to in done_queries:

2110 # Skip any prefetching, and any object preparation

2111 obj_list = done_queries[prefetch_to]

2112 continue

2113

2114 # Prepare objects:

2115 good_objects = True

2116 for obj in obj_list:

2117 # Since prefetching can re-use instances, it is possible to have

2118 # the same instance multiple times in obj_list, so obj might

2119 # already be prepared.

2120 if not hasattr(obj, "_prefetched_objects_cache"):

2121 try:

2122 obj._prefetched_objects_cache = {}

2123 except (AttributeError, TypeError):

2124 # Must be an immutable object from

2125 # values_list(flat=True), for example (TypeError) or

2126 # a QuerySet subclass that isn't returning Model

2127 # instances (AttributeError), either in Plain or a 3rd

2128 # party. prefetch_related() doesn't make sense, so quit.

2129 good_objects = False

2130 break

2131 if not good_objects:

2132 break

2133

2134 # Descend down tree

2135

2136 # We assume that objects retrieved are homogeneous (which is the premise

2137 # of prefetch_related), so what applies to first object applies to all.

2138 first_obj = obj_list[0]

2139 to_attr = lookup.get_current_to_attr(level)[0]

2140 prefetcher, descriptor, attr_found, is_fetched = get_prefetcher(

2141 first_obj, through_attr, to_attr

2142 )

2143

2144 if not attr_found:

2145 raise AttributeError(

2146 f"Cannot find '{through_attr}' on {first_obj.__class__.__name__} object, '{lookup.prefetch_through}' is an invalid "

2147 "parameter to prefetch_related()"

2148 )

2149

2150 if level == len(through_attrs) - 1 and prefetcher is None:

2151 # Last one, this *must* resolve to something that supports

2152 # prefetching, otherwise there is no point adding it and the

2153 # developer asking for it has made a mistake.

2154 raise ValueError(

2155 f"'{lookup.prefetch_through}' does not resolve to an item that supports "

2156 "prefetching - this is an invalid parameter to "

2157 "prefetch_related()."

2158 )

2159

2160 obj_to_fetch = None

2161 if prefetcher is not None:

2162 obj_to_fetch = [obj for obj in obj_list if not is_fetched(obj)]

2163

2164 if obj_to_fetch:

2165 obj_list, additional_lookups = prefetch_one_level(

2166 obj_to_fetch,

2167 prefetcher,

2168 lookup,

2169 level,

2170 )

2171 # We need to ensure we don't keep adding lookups from the

2172 # same relationships to stop infinite recursion. So, if we

2173 # are already on an automatically added lookup, don't add

2174 # the new lookups from relationships we've seen already.

2175 if not (

2176 prefetch_to in done_queries

2177 and lookup in auto_lookups

2178 and descriptor in followed_descriptors

2179 ):

2180 done_queries[prefetch_to] = obj_list

2181 new_lookups = normalize_prefetch_lookups(

2182 reversed(additional_lookups), prefetch_to

2183 )

2184 auto_lookups.update(new_lookups)

2185 all_lookups.extend(new_lookups)

2186 followed_descriptors.add(descriptor)

2187 else:

2188 # Either a singly related object that has already been fetched

2189 # (e.g. via select_related), or hopefully some other property

2190 # that doesn't support prefetching but needs to be traversed.

2191

2192 # We replace the current list of parent objects with the list

2193 # of related objects, filtering out empty or missing values so

2194 # that we can continue with nullable or reverse relations.

2195 new_obj_list = []

2196 for obj in obj_list:

2197 if through_attr in getattr(obj, "_prefetched_objects_cache", ()):

2198 # If related objects have been prefetched, use the

2199 # cache rather than the object's through_attr.

2200 new_obj = list(obj._prefetched_objects_cache.get(through_attr))

2201 else:

2202 try:

2203 new_obj = getattr(obj, through_attr)

2204 except exceptions.ObjectDoesNotExist:

2205 continue

2206 if new_obj is None:

2207 continue

2208 # We special-case `list` rather than something more generic

2209 # like `Iterable` because we don't want to accidentally match

2210 # user models that define __iter__.

2211 if isinstance(new_obj, list):

2212 new_obj_list.extend(new_obj)

2213 else:

2214 new_obj_list.append(new_obj)

2215 obj_list = new_obj_list

2216

2217

2218 def get_prefetcher(instance, through_attr, to_attr):

2219 """

2220 For the attribute 'through_attr' on the given instance, find

2221 an object that has a get_prefetch_queryset().

2222 Return a 4 tuple containing:

2223 (the object with get_prefetch_queryset (or None),

2224 the descriptor object representing this relationship (or None),

2225 a boolean that is False if the attribute was not found at all,

2226 a function that takes an instance and returns a boolean that is True if

2227 the attribute has already been fetched for that instance)

2228 """

2229

2230 def has_to_attr_attribute(instance):

2231 return hasattr(instance, to_attr)

2232

2233 prefetcher = None

2234 is_fetched = has_to_attr_attribute

2235

2236 # For singly related objects, we have to avoid getting the attribute

2237 # from the object, as this will trigger the query. So we first try

2238 # on the class, in order to get the descriptor object.

2239 rel_obj_descriptor = getattr(instance.__class__, through_attr, None)

2240 if rel_obj_descriptor is None:

2241 attr_found = hasattr(instance, through_attr)

2242 else:

2243 attr_found = True

2244 if rel_obj_descriptor:

2245 # singly related object, descriptor object has the

2246 # get_prefetch_queryset() method.

2247 if hasattr(rel_obj_descriptor, "get_prefetch_queryset"):

2248 prefetcher = rel_obj_descriptor

2249 is_fetched = rel_obj_descriptor.is_cached

2250 else:

2251 # descriptor doesn't support prefetching, so we go ahead and get

2252 # the attribute on the instance rather than the class to

2253 # support many related managers

2254 rel_obj = getattr(instance, through_attr)

2255 if hasattr(rel_obj, "get_prefetch_queryset"):

2256 prefetcher = rel_obj

2257 if through_attr != to_attr:

2258 # Special case cached_property instances because hasattr

2259 # triggers attribute computation and assignment.

2260 if isinstance(

2261 getattr(instance.__class__, to_attr, None), cached_property

2262 ):

2263

2264 def has_cached_property(instance):

2265 return to_attr in instance.__dict__

2266

2267 is_fetched = has_cached_property

2268 else:

2269

2270 def in_prefetched_cache(instance):

2271 return through_attr in instance._prefetched_objects_cache

2272

2273 is_fetched = in_prefetched_cache

2274 return prefetcher, rel_obj_descriptor, attr_found, is_fetched

2275

2276

2277 def prefetch_one_level(instances, prefetcher, lookup, level):

2278 """

2279 Helper function for prefetch_related_objects().

2280

2281 Run prefetches on all instances using the prefetcher object,

2282 assigning results to relevant caches in instance.

2283

2284 Return the prefetched objects along with any additional prefetches that

2285 must be done due to prefetch_related lookups found from default managers.

2286 """

2287 # prefetcher must have a method get_prefetch_queryset() which takes a list

2288 # of instances, and returns a tuple:

2289

2290 # (queryset of instances of self.model that are related to passed in instances,

2291 # callable that gets value to be matched for returned instances,

2292 # callable that gets value to be matched for passed in instances,

2293 # boolean that is True for singly related objects,

2294 # cache or field name to assign to,

2295 # boolean that is True when the previous argument is a cache name vs a field name).

2296

2297 # The 'values to be matched' must be hashable as they will be used

2298 # in a dictionary.

2299

2300 (

2301 rel_qs,

2302 rel_obj_attr,

2303 instance_attr,

2304 single,

2305 cache_name,

2306 is_descriptor,

2307 ) = prefetcher.get_prefetch_queryset(instances, lookup.get_current_queryset(level))

2308 # We have to handle the possibility that the QuerySet we just got back

2309 # contains some prefetch_related lookups. We don't want to trigger the

2310 # prefetch_related functionality by evaluating the query. Rather, we need

2311 # to merge in the prefetch_related lookups.

2312 # Copy the lookups in case it is a Prefetch object which could be reused

2313 # later (happens in nested prefetch_related).

2314 additional_lookups = [

2315 copy.copy(additional_lookup)

2316 for additional_lookup in getattr(rel_qs, "_prefetch_related_lookups", ())

2317 ]

2318 if additional_lookups:

2319 # Don't need to clone because the manager should have given us a fresh

2320 # instance, so we access an internal instead of using public interface

2321 # for performance reasons.

2322 rel_qs._prefetch_related_lookups = ()

2323

2324 all_related_objects = list(rel_qs)

2325

2326 rel_obj_cache = {}

2327 for rel_obj in all_related_objects:

2328 rel_attr_val = rel_obj_attr(rel_obj)

2329 rel_obj_cache.setdefault(rel_attr_val, []).append(rel_obj)

2330

2331 to_attr, as_attr = lookup.get_current_to_attr(level)

2332 # Make sure `to_attr` does not conflict with a field.

2333 if as_attr and instances:

2334 # We assume that objects retrieved are homogeneous (which is the premise

2335 # of prefetch_related), so what applies to first object applies to all.

2336 model = instances[0].__class__

2337 try:

2338 model._meta.get_field(to_attr)

2339 except exceptions.FieldDoesNotExist:

2340 pass

2341 else:

2342 msg = "to_attr={} conflicts with a field on the {} model."

2343 raise ValueError(msg.format(to_attr, model.__name__))

2344

2345 # Whether or not we're prefetching the last part of the lookup.

2346 leaf = len(lookup.prefetch_through.split(LOOKUP_SEP)) - 1 == level

2347

2348 for obj in instances:

2349 instance_attr_val = instance_attr(obj)

2350 vals = rel_obj_cache.get(instance_attr_val, [])

2351

2352 if single:

2353 val = vals[0] if vals else None

2354 if as_attr:

2355 # A to_attr has been given for the prefetch.

2356 setattr(obj, to_attr, val)

2357 elif is_descriptor:

2358 # cache_name points to a field name in obj.

2359 # This field is a descriptor for a related object.

2360 setattr(obj, cache_name, val)

2361 else:

2362 # No to_attr has been given for this prefetch operation and the

2363 # cache_name does not point to a descriptor. Store the value of

2364 # the field in the object's field cache.

2365 obj._state.fields_cache[cache_name] = val

2366 else:

2367 if as_attr:

2368 setattr(obj, to_attr, vals)

2369 else:

2370 manager = getattr(obj, to_attr)

2371 if leaf and lookup.queryset is not None:

2372 qs = manager._apply_rel_filters(lookup.queryset)

2373 else:

2374 qs = manager.get_queryset()

2375 qs._result_cache = vals

2376 # We don't want the individual qs doing prefetch_related now,

2377 # since we have merged this into the current work.

2378 qs._prefetch_done = True

2379 obj._prefetched_objects_cache[cache_name] = qs

2380 return all_related_objects, additional_lookups

2381

2382

2383 class RelatedPopulator:

2384 """

2385 RelatedPopulator is used for select_related() object instantiation.

2386

2387 The idea is that each select_related() model will be populated by a

2388 different RelatedPopulator instance. The RelatedPopulator instances get

2389 klass_info and select (computed in SQLCompiler) plus the used db as

2390 input for initialization. That data is used to compute which columns

2391 to use, how to instantiate the model, and how to populate the links

2392 between the objects.

2393

2394 The actual creation of the objects is done in populate() method. This

2395 method gets row and from_obj as input and populates the select_related()

2396 model instance.

2397 """

2398

2399 def __init__(self, klass_info, select, db):

2400 self.db = db

2401 # Pre-compute needed attributes. The attributes are:

2402 # - model_cls: the possibly deferred model class to instantiate

2403 # - either:

2404 # - cols_start, cols_end: usually the columns in the row are

2405 # in the same order model_cls.__init__ expects them, so we

2406 # can instantiate by model_cls(*row[cols_start:cols_end])

2407 # - reorder_for_init: When select_related descends to a child

2408 # class, then we want to reuse the already selected parent

2409 # data. However, in this case the parent data isn't necessarily

2410 # in the same order that Model.__init__ expects it to be, so

2411 # we have to reorder the parent data. The reorder_for_init

2412 # attribute contains a function used to reorder the field data

2413 # in the order __init__ expects it.

2414 # - pk_idx: the index of the primary key field in the reordered

2415 # model data. Used to check if a related object exists at all.

2416 # - init_list: the field attnames fetched from the database. For

2417 # deferred models this isn't the same as all attnames of the

2418 # model's fields.

2419 # - related_populators: a list of RelatedPopulator instances if

2420 # select_related() descends to related models from this model.

2421 # - local_setter, remote_setter: Methods to set cached values on

2422 # the object being populated and on the remote object. Usually

2423 # these are Field.set_cached_value() methods.

2424 select_fields = klass_info["select_fields"]

2425 from_parent = klass_info["from_parent"]

2426 if not from_parent:

2427 self.cols_start = select_fields[0]

2428 self.cols_end = select_fields[-1] + 1

2429 self.init_list = [

2430 f[0].target.attname for f in select[self.cols_start : self.cols_end]

2431 ]

2432 self.reorder_for_init = None

2433 else:

2434 attname_indexes = {

2435 select[idx][0].target.attname: idx for idx in select_fields

2436 }

2437 model_init_attnames = (

2438 f.attname for f in klass_info["model"]._meta.concrete_fields

2439 )

2440 self.init_list = [

2441 attname for attname in model_init_attnames if attname in attname_indexes

2442 ]

2443 self.reorder_for_init = operator.itemgetter(

2444 *[attname_indexes[attname] for attname in self.init_list]

2445 )

2446

2447 self.model_cls = klass_info["model"]

2448 self.pk_idx = self.init_list.index(self.model_cls._meta.pk.attname)

2449 self.related_populators = get_related_populators(klass_info, select, self.db)

2450 self.local_setter = klass_info["local_setter"]

2451 self.remote_setter = klass_info["remote_setter"]

2452

2453 def populate(self, row, from_obj):

2454 if self.reorder_for_init:

2455 obj_data = self.reorder_for_init(row)

2456 else:

2457 obj_data = row[self.cols_start : self.cols_end]

2458 if obj_data[self.pk_idx] is None:

2459 obj = None

2460 else:

2461 obj = self.model_cls.from_db(self.db, self.init_list, obj_data)

2462 for rel_iter in self.related_populators:

2463 rel_iter.populate(row, obj)

2464 self.local_setter(from_obj, obj)

2465 if obj is not None:

2466 self.remote_setter(obj, from_obj)

2467

2468

2469 def get_related_populators(klass_info, select, db):

2470 iterators = []

2471 related_klass_infos = klass_info.get("related_klass_infos", [])

2472 for rel_klass_info in related_klass_infos:

2473 rel_cls = RelatedPopulator(rel_klass_info, select, db)

2474 iterators.append(rel_cls)

2475 return iterators