Coverage for src/typedal/core.py: 100%

1"""

2 Core functionality of TypeDAL.

3"""

4 import contextlib

5 import csv

6 import datetime as dt

7 import inspect

8 import json

9 import math

10 import types

11 import typing

12 import warnings

13 from collections import defaultdict

14 from decimal import Decimal

15 from typing import Any, Optional

16

17 import pydal

18 from pydal._globals import DEFAULT

19 from pydal.objects import Field as _Field

20 from pydal.objects import Query as _Query

21 from pydal.objects import Row, Rows

22 from pydal.objects import Table as _Table

23 from typing_extensions import Self

24

25 from .helpers import (

26 DummyQuery,

27 all_annotations,

28 all_dict,

29 as_lambda,

30 extract_type_optional,

31 filter_out,

32 instanciate,

33 is_union,

34 looks_like,

35 mktable,

36 origin_is_subclass,

37 to_snake,

38 unwrap_type,

39)

40 from .types import (

41 CacheMetadata,

42 Expression,

43 Field,

44 Metadata,

45 PaginateDict,

46 Pagination,

47 Query,

48 _Types,

49)

50

51# use typing.cast(type, ...) to make mypy happy with unions

52 T_annotation = typing.Type[Any] | types.UnionType

53 T_Query = typing.Union["Table", Query, bool, None, "TypedTable", typing.Type["TypedTable"]]

54 T_Value = typing.TypeVar("T_Value") # actual type of the Field (via Generic)

55 T_MetaInstance = typing.TypeVar("T_MetaInstance", bound="TypedTable") # bound="TypedTable"; bound="TableMeta"

56 T = typing.TypeVar("T")

57

58 BASIC_MAPPINGS: dict[T_annotation, str] = {

59 str: "string",

60 int: "integer",

61 bool: "boolean",

62 bytes: "blob",

63 float: "double",

64 object: "json",

65 Decimal: "decimal(10,2)",

66 dt.date: "date",

67 dt.time: "time",

68 dt.datetime: "datetime",

69}

70

71

72 def is_typed_field(cls: Any) -> typing.TypeGuard["TypedField[Any]"]:

73 """

74 Is `cls` an instance or subclass of TypedField?

75

76 Deprecated

77 """

78 return (

79 isinstance(cls, TypedField)

80 or isinstance(typing.get_origin(cls), type)

81 and issubclass(typing.get_origin(cls), TypedField)

82 )

83

84

85 JOIN_OPTIONS = typing.Literal["left", "inner", None]

86 DEFAULT_JOIN_OPTION: JOIN_OPTIONS = "left"

87

88# table-ish paramter:

89 P_Table = typing.Union[typing.Type["TypedTable"], pydal.objects.Table]

90

91 Condition: typing.TypeAlias = typing.Optional[

92 typing.Callable[

93 # self, other -> Query

94 [P_Table, P_Table],

95 Query | bool,

96 ]

97]

98

99 OnQuery: typing.TypeAlias = typing.Optional[

100 typing.Callable[

101 # self, other -> list of .on statements

102 [P_Table, P_Table],

103 list[Expression],

104 ]

105]

106

107 To_Type = typing.TypeVar("To_Type", type[Any], typing.Type[Any], str)

108

109

110 class Relationship(typing.Generic[To_Type]):

111 """

112 Define a relationship to another table.

113 """

114

115 _type: To_Type

116 table: typing.Type["TypedTable"] | type | str

117 condition: Condition

118 on: OnQuery

119 multiple: bool

120 join: JOIN_OPTIONS

121

122 def __init__(

123 self,

124 _type: To_Type,

125 condition: Condition = None,

126 join: JOIN_OPTIONS = None,

127 on: OnQuery = None,

128 ):

129 """

130 Should not be called directly, use relationship() instead!

131 """

132 if condition and on:

133 warnings.warn(f"Relation | Both specified! {condition=} {on=} {_type=}")

134 raise ValueError("Please specify either a condition or an 'on' statement for this relationship!")

135

136 self._type = _type

137 self.condition = condition

138 self.join = "left" if on else join # .on is always left join!

139 self.on = on

140

141 if args := typing.get_args(_type):

142 self.table = unwrap_type(args[0])

143 self.multiple = True

144 else:

145 self.table = _type

146 self.multiple = False

147

148 if isinstance(self.table, str):

149 self.table = TypeDAL.to_snake(self.table)

150

151 def clone(self, **update: Any) -> "Relationship[To_Type]":

152 """

153 Create a copy of the relationship, possibly updated.

154 """

155 return self.__class__(

156 update.get("_type") or self._type,

157 update.get("condition") or self.condition,

158 update.get("join") or self.join,

159 update.get("on") or self.on,

160 )

161

162 def __repr__(self) -> str:

163 """

164 Representation of the relationship.

165 """

166 if callback := self.condition or self.on:

167 src_code = inspect.getsource(callback).strip()

168 else:

169 cls_name = self._type if isinstance(self._type, str) else self._type.__name__ # type: ignore

170 src_code = f"to {cls_name} (missing condition)"

171

172 join = f":{self.join}" if self.join else ""

173 return f"<Relationship{join} {src_code}>"

174

175 def get_table(self, db: "TypeDAL") -> typing.Type["TypedTable"]:

176 """

177 Get the table this relationship is bound to.

178 """

179 table = self.table # can be a string because db wasn't available yet

180 if isinstance(table, str):

181 if mapped := db._class_map.get(table):

182 # yay

183 return mapped

184

185 # boo, fall back to untyped table but pretend it is typed:

186 return typing.cast(typing.Type["TypedTable"], db[table]) # eh close enough!

187

188 return table

189

190 def get_table_name(self) -> str:

191 """

192 Get the name of the table this relationship is bound to.

193 """

194 if isinstance(self.table, str):

195 return self.table

196

197 if isinstance(self.table, pydal.objects.Table):

198 return str(self.table)

199

200 # else: typed table

201 try:

202 table = self.table._ensure_table_defined() if issubclass(self.table, TypedTable) else self.table

203 except Exception: # pragma: no cover

204 table = self.table

205

206 return str(table)

207

208 def __get__(self, instance: Any, owner: Any) -> typing.Optional[list[Any]] | "Relationship[To_Type]":

209 """

210 Relationship is a descriptor class, which can be returned from a class but not an instance.

211

212 For an instance, using .join() will replace the Relationship with the actual data.

213 If you forgot to join, a warning will be shown and empty data will be returned.

214 """

215 if not instance:

216 # relationship queried on class, that's allowed

217 return self

218

219 warnings.warn(

220 "Trying to get data from a relationship object! Did you forget to join it?", category=RuntimeWarning

221 )

222 if self.multiple:

223 return []

224 else:

225 return None

226

227

228 def relationship(

229 _type: To_Type, condition: Condition = None, join: JOIN_OPTIONS = None, on: OnQuery = None

230) -> Relationship[To_Type]:

231 """

232 Define a relationship to another table, when its id is not stored in the current table.

233

234 Example:

235 class User(TypedTable):

236 name: str

237

238 posts = relationship(list["Post"], condition=lambda self, post: self.id == post.author, join='left')

239

240 class Post(TypedTable):

241 title: str

242 author: User

243

244 User.join("posts").first() # User instance with list[Post] in .posts

245

246 Here, Post stores the User ID, but `relationship(list["Post"])` still allows you to get the user's posts.

247 In this case, the join strategy is set to LEFT so users without posts are also still selected.

248

249 For complex queries with a pivot table, a `on` can be set insteaad of `condition`:

250 class User(TypedTable):

251 ...

252

253 tags = relationship(list["Tag"], on=lambda self, tag: [

254 Tagged.on(Tagged.entity == entity.gid),

255 Tag.on((Tagged.tag == tag.id)),

256 ])

257

258 If you'd try to capture this in a single 'condition', pydal would create a cross join which is much less efficient.

259 """

260 return Relationship(_type, condition, join, on)

261

262

263 def _generate_relationship_condition(

264 _: typing.Type["TypedTable"], key: str, field: typing.Union["TypedField[Any]", "Table", typing.Type["TypedTable"]]

265) -> Condition:

266 origin = typing.get_origin(field)

267 # else: generic

268

269 if origin == list:

270 # field = typing.get_args(field)[0] # actual field

271 # return lambda _self, _other: cls[key].contains(field)

272

273 return lambda _self, _other: _self[key].contains(_other.id)

274 else:

275 # normal reference

276 # return lambda _self, _other: cls[key] == field.id

277 return lambda _self, _other: _self[key] == _other.id

278

279

280 def to_relationship(

281 cls: typing.Type["TypedTable"] | type[Any],

282 key: str,

283 field: typing.Union["TypedField[Any]", "Table", typing.Type["TypedTable"]],

284) -> typing.Optional[Relationship[Any]]:

285 """

286 Used to automatically create relationship instance for reference fields.

287

288 Example:

289 class MyTable(TypedTable):

290 reference: OtherTable

291

292 `reference` contains the id of an Other Table row.

293 MyTable.relationships should have 'reference' as a relationship, so `MyTable.join('reference')` should work.

294

295 This function will automatically perform this logic (called in db.define):

296 to_relationship(MyTable, 'reference', OtherTable) -> Relationship[OtherTable]

297

298 Also works for list:reference (list[OtherTable]) and TypedField[OtherTable].

299 """

300 if looks_like(field, TypedField):

301 if args := typing.get_args(field):

302 field = args[0]

303 else:

304 # weird

305 return None

306

307 field, optional = extract_type_optional(field)

308

309 try:

310 condition = _generate_relationship_condition(cls, key, field)

311 except Exception as e: # pragma: no cover

312 warnings.warn("Could not generate Relationship condition", source=e)

313 condition = None

314

315 if not condition: # pragma: no cover

316 # something went wrong, not a valid relationship

317 warnings.warn(f"Invalid relationship for {cls.__name__}.{key}: {field}")

318 return None

319

320 join = "left" if optional or typing.get_origin(field) == list else "inner"

321

322 return Relationship(typing.cast(type[TypedTable], field), condition, typing.cast(JOIN_OPTIONS, join))

323

324

325 class TypeDAL(pydal.DAL): # type: ignore

326 """

327 Drop-in replacement for pyDAL with layer to convert class-based table definitions to classical pydal define_tables.

328 """

329

330 def __init__(

331 self,

332 uri: str = "sqlite://dummy.db",

333 pool_size: int = 0,

334 folder: Optional[str] = None,

335 db_codec: str = "UTF-8",

336 check_reserved: Optional[list[str]] = None,

337 migrate: bool = True,

338 fake_migrate: bool = False,

339 migrate_enabled: bool = True,

340 fake_migrate_all: bool = False,

341 decode_credentials: bool = False,

342 driver_args: Optional[dict[str, Any]] = None,

343 adapter_args: Optional[dict[str, Any]] = None,

344 attempts: int = 5,

345 auto_import: bool = False,

346 bigint_id: bool = False,

347 debug: bool = False,

348 lazy_tables: bool = False,

349 db_uid: Optional[str] = None,

350 after_connection: typing.Callable[..., Any] = None,

351 tables: Optional[list[str]] = None,

352 ignore_field_case: bool = True,

353 entity_quoting: bool = True,

354 table_hash: Optional[str] = None,

355 enable_typedal_caching: bool = True,

356 ) -> None:

357 """

358 Adds some internal tables after calling pydal's default init.

359

360 Set enable_typedal_caching to False to disable this behavior.

361 """

362 super().__init__(

363 uri,

364 pool_size,

365 folder,

366 db_codec,

367 check_reserved,

368 migrate,

369 fake_migrate,

370 migrate_enabled,

371 fake_migrate_all,

372 decode_credentials,

373 driver_args,

374 adapter_args,

375 attempts,

376 auto_import,

377 bigint_id,

378 debug,

379 lazy_tables,

380 db_uid,

381 after_connection,

382 tables,

383 ignore_field_case,

384 entity_quoting,

385 table_hash,

386 )

387

388 if enable_typedal_caching:

389 self.define(_TypedalCache, migrate=True)

390 self.define(_TypedalCacheDependency, migrate=True)

391

392 default_kwargs: typing.ClassVar[typing.Dict[str, Any]] = {

393 # fields are 'required' (notnull) by default:

394 "notnull": True,

395 }

396

397 # maps table name to typedal class, for resolving future references

398 _class_map: typing.ClassVar[dict[str, typing.Type["TypedTable"]]] = {}

399

400 def _define(self, cls: typing.Type[T], **kwargs: Any) -> typing.Type[T]:

401 # todo: new relationship item added should also invalidate (previously unrelated) cache result

402

403 # todo: option to enable/disable cache dependency behavior:

404 # - don't set _before_update and _before_delete

405 # - don't add TypedalCacheDependency entry

406 # - don't invalidate other item on new row of this type

407

408 # when __future__.annotations is implemented, cls.__annotations__ will not work anymore as below.

409 # proper way to handle this would be (but gives error right now due to Table implementing magic methods):

410 # typing.get_type_hints(cls, globalns=None, localns=None)

411

412 # dirty way (with evil eval):

413 # [eval(v) for k, v in cls.__annotations__.items()]

414 # this however also stops working when variables outside this scope or even references to other

415 # objects are used. So for now, this package will NOT work when from __future__ import annotations is used,

416 # and might break in the future, when this annotations behavior is enabled by default.

417

418 # non-annotated variables have to be passed to define_table as kwargs

419 full_dict = all_dict(cls) # includes properties from parents (e.g. useful for mixins)

420

421 tablename = self.to_snake(cls.__name__)

422 # grab annotations of cls and it's parents:

423 annotations = all_annotations(cls)

424 # extend with `prop = TypedField()` 'annotations':

425 annotations |= {k: typing.cast(type, v) for k, v in full_dict.items() if is_typed_field(v)}

426 # remove internal stuff:

427 annotations = {k: v for k, v in annotations.items() if not k.startswith("_")}

428

429 typedfields: dict[str, TypedField[Any]] = {

430 k: instanciate(v, True) for k, v in annotations.items() if is_typed_field(v)

431 }

432

433 relationships: dict[str, type[Relationship[Any]]] = filter_out(annotations, Relationship)

434

435 fields = {fname: self._to_field(fname, ftype) for fname, ftype in annotations.items()}

436

437 # ! dont' use full_dict here:

438 other_kwargs = kwargs | {

439 k: v for k, v in cls.__dict__.items() if k not in annotations and not k.startswith("_")

440 }

441

442 for key in typedfields.keys() - full_dict.keys():

443 # typed fields that don't haven't been added to the object yet

444 setattr(cls, key, typedfields[key])

445

446 # start with base classes and overwrite with current class:

447 relationships = filter_out(full_dict, Relationship) | relationships | filter_out(other_kwargs, Relationship)

448

449 # DEPRECATED: Relationship as annotation is currently not supported!

450 # ensure they are all instances and

451 # not mix of instances (`= relationship()`) and classes (`: Relationship[...]`):

452 # relationships = {

453 # k: v if isinstance(v, Relationship) else to_relationship(cls, k, v) for k, v in relationships.items()

454 # }

455

456 # keys of implicit references (also relationships):

457 reference_field_keys = [k for k, v in fields.items() if v.type.split(" ")[0] in ("list:reference", "reference")]

458

459 # add implicit relationships:

460 # User; list[User]; TypedField[User]; TypedField[list[User]]

461 relationships |= {

462 k: new_relationship

463 for k in reference_field_keys

464 if k not in relationships and (new_relationship := to_relationship(cls, k, annotations[k]))

465 }

466

467 table: Table = self.define_table(tablename, *fields.values(), **other_kwargs)

468

469 for name, typed_field in typedfields.items():

470 field = fields[name]

471 typed_field.bind(field, table)

472

473 if issubclass(cls, TypedTable):

474 cls.__set_internals__(

475 db=self,

476 table=table,

477 # by now, all relationships should be instances!

478 relationships=typing.cast(dict[str, Relationship[Any]], relationships),

479 )

480 self._class_map[str(table)] = cls

481 else:

482 warnings.warn("db.define used without inheriting TypedTable. This could lead to strange problems!")

483

484 if not tablename.startswith("typedal_"):

485 # todo: config

486 table._before_update.append(lambda s, _: _remove_cache(s, tablename))

487 table._before_delete.append(lambda s: _remove_cache(s, tablename))

488

489 return cls

490

491 @typing.overload

492 def define(self, maybe_cls: None = None, **kwargs: Any) -> typing.Callable[[typing.Type[T]], typing.Type[T]]:

493 """

494 Typing Overload for define without a class.

495

496 @db.define()

497 class MyTable(TypedTable): ...

498 """

499

500 @typing.overload

501 def define(self, maybe_cls: typing.Type[T], **kwargs: Any) -> typing.Type[T]:

502 """

503 Typing Overload for define with a class.

504

505 @db.define

506 class MyTable(TypedTable): ...

507 """

508

509 def define(

510 self, maybe_cls: typing.Type[T] | None = None, **kwargs: Any

511 ) -> typing.Type[T] | typing.Callable[[typing.Type[T]], typing.Type[T]]:

512 """

513 Can be used as a decorator on a class that inherits `TypedTable`, \

514 or as a regular method if you need to define your classes before you have access to a 'db' instance.

515

516 You can also pass extra arguments to db.define_table.

517 See http://www.web2py.com/books/default/chapter/29/06/the-database-abstraction-layer#Table-constructor

518

519 Example:

520 @db.define

521 class Person(TypedTable):

522 ...

523

524 class Article(TypedTable):

525 ...

526

527 # at a later time:

528 db.define(Article)

529

530 Returns:

531 the result of pydal.define_table

532 """

533

534 def wrapper(cls: typing.Type[T]) -> typing.Type[T]:

535 return self._define(cls, **kwargs)

536

537 if maybe_cls:

538 return wrapper(maybe_cls)

539

540 return wrapper

541

542 # def drop(self, table_name: str) -> None:

543 # """

544 # Remove a table by name (both on the database level and the typedal level).

545 # """

546 # # drop calls TypedTable.drop() and removes it from the `_class_map`

547 # if cls := self._class_map.pop(table_name, None):

548 # cls.drop()

549

550 # def drop_all(self, max_retries: int = None) -> None:

551 # """

552 # Remove all tables and keep doing so until everything is gone!

553 # """

554 # retries = 0

555 # if max_retries is None:

556 # max_retries = len(self.tables)

557 #

558 # while self.tables:

559 # retries += 1

560 # for table in self.tables:

561 # self.drop(table)

562 #

563 # if retries > max_retries:

564 # raise RuntimeError("Could not delete all tables")

565

566 def __call__(self, *_args: T_Query, **kwargs: Any) -> "TypedSet":

567 """

568 A db instance can be called directly to perform a query.

569

570 Usually, only a query is passed.

571

572 Example:

573 db(query).select()

574

575 """

576 args = list(_args)

577 if args:

578 cls = args[0]

579 if isinstance(cls, bool):

580 raise ValueError("Don't actually pass a bool to db()! Use a query instead.")

581

582 if isinstance(cls, type) and issubclass(type(cls), type) and issubclass(cls, TypedTable):

583 # table defined without @db.define decorator!

584 _cls: typing.Type[TypedTable] = cls

585 args[0] = _cls.id != None

586

587 _set = super().__call__(*args, **kwargs)

588 return typing.cast(TypedSet, _set)

589

590 @classmethod

591 def _build_field(cls, name: str, _type: str, **kw: Any) -> Field:

592 return Field(name, _type, **{**cls.default_kwargs, **kw})

593

594 @classmethod

595 def _annotation_to_pydal_fieldtype(

596 cls, _ftype: T_annotation, mut_kw: typing.MutableMapping[str, Any]

597 ) -> Optional[str]:

598 # ftype can be a union or type. typing.cast is sometimes used to tell mypy when it's not a union.

599 ftype = typing.cast(type, _ftype) # cast from typing.Type to type to make mypy happy)

600

601 if isinstance(ftype, str):

602 # extract type from string

603 ftype = typing.get_args(typing.Type[ftype])[0]._evaluate(

604 localns=locals(), globalns=globals(), recursive_guard=frozenset()

605 )

606

607 if mapping := BASIC_MAPPINGS.get(ftype):

608 # basi types

609 return mapping

610 elif isinstance(ftype, _Table):

611 # db.table

612 return f"reference {ftype._tablename}"

613 elif issubclass(type(ftype), type) and issubclass(ftype, TypedTable):

614 # SomeTable

615 snakename = cls.to_snake(ftype.__name__)

616 return f"reference {snakename}"

617 elif isinstance(ftype, TypedField):

618 # FieldType(type, ...)

619 return ftype._to_field(mut_kw)

620 elif origin_is_subclass(ftype, TypedField):

621 # TypedField[int]

622 return cls._annotation_to_pydal_fieldtype(typing.get_args(ftype)[0], mut_kw)

623 elif isinstance(ftype, types.GenericAlias) and typing.get_origin(ftype) in (list, TypedField):

624 # list[str] -> str -> string -> list:string

625 _child_type = typing.get_args(ftype)[0]

626 _child_type = cls._annotation_to_pydal_fieldtype(_child_type, mut_kw)

627 return f"list:{_child_type}"

628 elif is_union(ftype):

629 # str | int -> UnionType

630 # typing.Union[str | int] -> typing._UnionGenericAlias

631

632 # Optional[type] == type | None

633

634 match typing.get_args(ftype):

635 case (_child_type, _Types.NONETYPE) | (_Types.NONETYPE, _child_type):

636 # good union of Nullable

637

638 # if a field is optional, it is nullable:

639 mut_kw["notnull"] = False

640 return cls._annotation_to_pydal_fieldtype(_child_type, mut_kw)

641 case _:

642 # two types is not supported by the db!

643 return None

644 else:

645 return None

646

647 @classmethod

648 def _to_field(cls, fname: str, ftype: type, **kw: Any) -> Field:

649 """

650 Convert a annotation into a pydal Field.

651

652 Args:

653 fname: name of the property

654 ftype: annotation of the property

655 kw: when using TypedField or a function returning it (e.g. StringField),

656 keyword args can be used to pass any other settings you would normally to a pydal Field

657

658 -> pydal.Field(fname, ftype, **kw)

659

660 Example:

661 class MyTable:

662 fname: ftype

663 id: int

664 name: str

665 reference: Table

666 other: TypedField(str, default="John Doe") # default will be in kwargs

667 """

668 fname = cls.to_snake(fname)

669

670 if converted_type := cls._annotation_to_pydal_fieldtype(ftype, kw):

671 return cls._build_field(fname, converted_type, **kw)

672 else:

673 raise NotImplementedError(f"Unsupported type {ftype}/{type(ftype)}")

674

675 @staticmethod

676 def to_snake(camel: str) -> str:

677 """

678 Moved to helpers, kept as a static method for legacy reasons.

679 """

680 return to_snake(camel)

681

682

683 class TableProtocol(typing.Protocol): # pragma: no cover

684 """

685 Make mypy happy.

686 """

687

688 id: int # noqa: A003

689

690 def __getitem__(self, item: str) -> Field:

691 """

692 Tell mypy a Table supports dictionary notation for columns.

693 """

694

695

696 class Table(_Table, TableProtocol): # type: ignore

697 """

698 Make mypy happy.

699 """

700

701

702 class TableMeta(type):

703 """

704 This metaclass contains functionality on table classes, that doesn't exist on its instances.

705

706 Example:

707 class MyTable(TypedTable):

708 some_field: TypedField[int]

709

710 MyTable.update_or_insert(...) # should work

711

712 MyTable.some_field # -> Field, can be used to query etc.

713

714 row = MyTable.first() # returns instance of MyTable

715

716 # row.update_or_insert(...) # shouldn't work!

717

718 row.some_field # -> int, with actual data

719

720 """

721

722 # set up by db.define:

723 # _db: TypeDAL | None = None

724 # _table: Table | None = None

725 _db: TypeDAL | None = None

726 _table: Table | None = None

727 _relationships: dict[str, Relationship[Any]] | None = None

728

729 #########################

730 # TypeDAL custom logic: #

731 #########################

732

733 def __set_internals__(self, db: pydal.DAL, table: Table, relationships: dict[str, Relationship[Any]]) -> None:

734 """

735 Store the related database and pydal table for later usage.

736 """

737 self._db = db

738 self._table = table

739 self._relationships = relationships

740

741 def __getattr__(self, col: str) -> Optional[Field]:

742 """

743 Magic method used by TypedTableMeta to get a database field with dot notation on a class.

744

745 Example:

746 SomeTypedTable.col -> db.table.col (via TypedTableMeta.__getattr__)

747

748 """

749 if self._table:

750 return getattr(self._table, col, None)

751

752 return None

753

754 def _ensure_table_defined(self) -> Table:

755 if not self._table:

756 raise EnvironmentError("@define or db.define is not called on this class yet!")

757 return self._table

758

759 def __iter__(self) -> typing.Generator[Field, None, None]:

760 """

761 Loop through the columns of this model.

762 """

763 table = self._ensure_table_defined()

764 yield from iter(table)

765

766 def __getitem__(self, item: str) -> Field:

767 """

768 Allow dict notation to get a column of this table (-> Field instance).

769 """

770 table = self._ensure_table_defined()

771 return table[item]

772

773 def __str__(self) -> str:

774 """

775 Normally, just returns the underlying table name, but with a fallback if the model is unbound.

776 """

777 if self._table:

778 return str(self._table)

779 else:

780 return f"<unbound table {self.__name__}>"

781

782 def from_row(self: typing.Type[T_MetaInstance], row: pydal.objects.Row) -> T_MetaInstance:

783 """

784 Create a model instance from a pydal row.

785 """

786 return self(row)

787

788 def all(self: typing.Type[T_MetaInstance]) -> "TypedRows[T_MetaInstance]": # noqa: A003

789 """

790 Return all rows for this model.

791 """

792 return self.collect()

793

794 def __json__(self: typing.Type[T_MetaInstance], instance: T_MetaInstance | None = None) -> dict[str, Any]:

795 """

796 Convert to a json-dumpable dict.

797

798 as_dict is not fully json-dumpable, so use as_json and json.loads to ensure it is dumpable (and loadable).

799 todo: can this be optimized?

800

801 See Also:

802 https://github.com/jeff-hykin/json_fix

803 """

804 string = instance.as_json() if instance else self.as_json()

805

806 return typing.cast(dict[str, Any], json.loads(string))

807

808 def get_relationships(self) -> dict[str, Relationship[Any]]:

809 """

810 Return the registered relationships of the current model.

811 """

812 return self._relationships or {}

813

814 ##########################

815 # TypeDAL Modified Logic #

816 ##########################

817

818 def insert(self: typing.Type[T_MetaInstance], **fields: Any) -> T_MetaInstance:

819 """

820 This is only called when db.define is not used as a decorator.

821

822 cls.__table functions as 'self'

823

824 Args:

825 **fields: anything you want to insert in the database

826

827 Returns: the ID of the new row.

828

829 """

830 table = self._ensure_table_defined()

831

832 result = table.insert(**fields)

833 # it already is an int but mypy doesn't understand that

834 return self(result)

835

836 def _insert(self, **fields: Any) -> str:

837 table = self._ensure_table_defined()

838

839 return str(table._insert(**fields))

840

841 def bulk_insert(self: typing.Type[T_MetaInstance], items: list[dict[str, Any]]) -> "TypedRows[T_MetaInstance]":

842 """

843 Insert multiple rows, returns a TypedRows set of new instances.

844 """

845 table = self._ensure_table_defined()

846 result = table.bulk_insert(items)

847 return self.where(lambda row: row.id.belongs(result)).collect()

848

849 def update_or_insert(

850 self: typing.Type[T_MetaInstance], query: T_Query | dict[str, Any] = DEFAULT, **values: Any

851 ) -> T_MetaInstance:

852 """

853 Update a row if query matches, else insert a new one.

854

855 Returns the created or updated instance.

856 """

857 table = self._ensure_table_defined()

858

859 if query is DEFAULT:

860 record = table(**values)

861 elif isinstance(query, dict):

862 record = table(**query)

863 else:

864 record = table(query)

865

866 if not record:

867 return self.insert(**values)

868

869 record.update_record(**values)

870 return self(record)

871

872 def validate_and_insert(

873 self: typing.Type[T_MetaInstance], **fields: Any

874 ) -> tuple[Optional[T_MetaInstance], Optional[dict[str, str]]]:

875 """

876 Validate input data and then insert a row.

877

878 Returns a tuple of (the created instance, a dict of errors).

879 """

880 table = self._ensure_table_defined()

881 result = table.validate_and_insert(**fields)

882 if row_id := result.get("id"):

883 return self(row_id), None

884 else:

885 return None, result.get("errors")

886

887 def validate_and_update(

888 self: typing.Type[T_MetaInstance], query: Query, **fields: Any

889 ) -> tuple[Optional[T_MetaInstance], Optional[dict[str, str]]]:

890 """

891 Validate input data and then update max 1 row.

892

893 Returns a tuple of (the updated instance, a dict of errors).

894 """

895 table = self._ensure_table_defined()

896

897 try:

898 result = table.validate_and_update(query, **fields)

899 except Exception as e:

900 result = {"errors": {"exception": str(e)}}

901

902 if errors := result.get("errors"):

903 return None, errors

904 elif row_id := result.get("id"):

905 return self(row_id), None

906 else: # pragma: no cover

907 # update on query without result (shouldnt happen)

908 return None, None

909

910 def validate_and_update_or_insert(

911 self: typing.Type[T_MetaInstance], query: Query, **fields: Any

912 ) -> tuple[Optional[T_MetaInstance], Optional[dict[str, str]]]:

913 """

914 Validate input data and then update_and_insert (on max 1 row).

915

916 Returns a tuple of (the updated/created instance, a dict of errors).

917 """

918 table = self._ensure_table_defined()

919 result = table.validate_and_update_or_insert(query, **fields)

920

921 if errors := result.get("errors"):

922 return None, errors

923 elif row_id := result.get("id"):

924 return self(row_id), None

925 else: # pragma: no cover

926 # update on query without result (shouldnt happen)

927 return None, None

928

929 def select(self: typing.Type[T_MetaInstance], *a: Any, **kw: Any) -> "QueryBuilder[T_MetaInstance]":

930 """

931 See QueryBuilder.select!

932 """

933 return QueryBuilder(self).select(*a, **kw)

934

935 def paginate(self: typing.Type[T_MetaInstance], limit: int, page: int = 1) -> "PaginatedRows[T_MetaInstance]":

936 """

937 See QueryBuilder.paginate!

938 """

939 return QueryBuilder(self).paginate(limit=limit, page=page)

940

941 def chunk(

942 self: typing.Type[T_MetaInstance], chunk_size: int

943 ) -> typing.Generator["TypedRows[T_MetaInstance]", Any, None]:

944 """

945 See QueryBuilder.chunk!

946 """

947 return QueryBuilder(self).chunk(chunk_size)

948

949 def where(self: typing.Type[T_MetaInstance], *a: Any, **kw: Any) -> "QueryBuilder[T_MetaInstance]":

950 """

951 See QueryBuilder.where!

952 """

953 return QueryBuilder(self).where(*a, **kw)

954

955 def cache(self: typing.Type[T_MetaInstance], *deps: Any, **kwargs: Any) -> "QueryBuilder[T_MetaInstance]":

956 """

957 See QueryBuilder.cache!

958 """

959 return QueryBuilder(self).cache(*deps, **kwargs)

960

961 def count(self: typing.Type[T_MetaInstance]) -> int:

962 """

963 See QueryBuilder.count!

964 """

965 return QueryBuilder(self).count()

966

967 def first(self: typing.Type[T_MetaInstance]) -> T_MetaInstance | None:

968 """

969 See QueryBuilder.first!

970 """

971 return QueryBuilder(self).first()

972

973 def join(

974 self: typing.Type[T_MetaInstance],

975 *fields: str | typing.Type["TypedTable"],

976 method: JOIN_OPTIONS = None,

977 on: OnQuery | list[Expression] | Expression = None,

978 condition: Condition = None,

979 ) -> "QueryBuilder[T_MetaInstance]":

980 """

981 See QueryBuilder.join!

982 """

983 return QueryBuilder(self).join(*fields, on=on, condition=condition, method=method)

984

985 def collect(self: typing.Type[T_MetaInstance], verbose: bool = False) -> "TypedRows[T_MetaInstance]":

986 """

987 See QueryBuilder.collect!

988 """

989 return QueryBuilder(self).collect(verbose=verbose)

990

991 @property

992 def ALL(cls) -> pydal.objects.SQLALL:

993 """

994 Select all fields for this table.

995 """

996 table = cls._ensure_table_defined()

997

998 return table.ALL

999

1000 ##########################

1001 # TypeDAL Shadowed Logic #

1002 ##########################

1003 fields: list[str]

1004

1005 # other table methods:

1006

1007 def drop(self, mode: str = "") -> None:

1008 """

1009 Remove the underlying table.

1010 """

1011 table = self._ensure_table_defined()

1012 table.drop(mode)

1013

1014 def create_index(self, name: str, *fields: Field | str, **kwargs: Any) -> bool:

1015 """

1016 Add an index on some columns of this table.

1017 """

1018 table = self._ensure_table_defined()

1019 result = table.create_index(name, *fields, **kwargs)

1020 return typing.cast(bool, result)

1021

1022 def drop_index(self, name: str, if_exists: bool = False) -> bool:

1023 """

1024 Remove an index from this table.

1025 """

1026 table = self._ensure_table_defined()

1027 result = table.drop_index(name, if_exists)

1028 return typing.cast(bool, result)

1029

1030 def import_from_csv_file(

1031 self,

1032 csvfile: typing.TextIO,

1033 id_map: dict[str, str] = None,

1034 null: Any = "<NULL>",

1035 unique: str = "uuid",

1036 id_offset: dict[str, int] = None, # id_offset used only when id_map is None

1037 transform: typing.Callable[[dict[Any, Any]], dict[Any, Any]] = None,

1038 validate: bool = False,

1039 encoding: str = "utf-8",

1040 delimiter: str = ",",

1041 quotechar: str = '"',

1042 quoting: int = csv.QUOTE_MINIMAL,

1043 restore: bool = False,

1044 **kwargs: Any,

1045 ) -> None:

1046 """

1047 Load a csv file into the database.

1048 """

1049 table = self._ensure_table_defined()

1050 table.import_from_csv_file(

1051 csvfile,

1052 id_map=id_map,

1053 null=null,

1054 unique=unique,

1055 id_offset=id_offset,

1056 transform=transform,

1057 validate=validate,

1058 encoding=encoding,

1059 delimiter=delimiter,

1060 quotechar=quotechar,

1061 quoting=quoting,

1062 restore=restore,

1063 **kwargs,

1064 )

1065

1066 def on(self, query: Query | bool) -> Expression:

1067 """

1068 Shadow Table.on.

1069

1070 Used for joins.

1071

1072 See Also:

1073 http://web2py.com/books/default/chapter/29/06/the-database-abstraction-layer?search=export_to_csv_file#One-to-many-relation

1074 """

1075 table = self._ensure_table_defined()

1076 return typing.cast(Expression, table.on(query))

1077

1078 def with_alias(self, alias: str) -> _Table:

1079 """

1080 Shadow Table.with_alias.

1081

1082 Useful for joins when joining the same table multiple times.

1083

1084 See Also:

1085 http://web2py.com/books/default/chapter/29/06/the-database-abstraction-layer?search=export_to_csv_file#One-to-many-relation

1086 """

1087 table = self._ensure_table_defined()

1088 return table.with_alias(alias)

1089

1090 # @typing.dataclass_transform()

1091

1092

1093 class TypedField(typing.Generic[T_Value]): # pragma: no cover

1094 """

1095 Typed version of pydal.Field, which will be converted to a normal Field in the background.

1096 """

1097

1098 # will be set by .bind on db.define

1099 name = ""

1100 _db: Optional[pydal.DAL] = None

1101 _rname: Optional[str] = None

1102 _table: Optional[Table] = None

1103 _field: Optional[Field] = None

1104

1105 _type: T_annotation

1106 kwargs: Any

1107

1108 def __init__(self, _type: typing.Type[T_Value] | types.UnionType = str, /, **settings: Any) -> None: # type: ignore

1109 """

1110 A TypedFieldType should not be inited manually, but TypedField (from `fields.py`) should be used!

1111 """

1112 self._type = _type

1113 self.kwargs = settings

1114 super().__init__()

1115

1116 @typing.overload

1117 def __get__(self, instance: T_MetaInstance, owner: typing.Type[T_MetaInstance]) -> T_Value: # pragma: no cover

1118 """

1119 row.field -> (actual data).

1120 """

1121

1122 @typing.overload

1123 def __get__(self, instance: None, owner: "typing.Type[TypedTable]") -> "TypedField[T_Value]": # pragma: no cover

1124 """

1125 Table.field -> Field.

1126 """

1127

1128 def __get__(

1129 self, instance: T_MetaInstance | None, owner: typing.Type[T_MetaInstance]

1130 ) -> typing.Union[T_Value, "TypedField[T_Value]"]:

1131 """

1132 Since this class is a Descriptor field, \

1133 it returns something else depending on if it's called on a class or instance.

1134

1135 (this is mostly for mypy/typing)

1136 """

1137 if instance:

1138 # this is only reached in a very specific case:

1139 # an instance of the object was created with a specific set of fields selected (excluding the current one)

1140 # in that case, no value was stored in the owner -> return None (since the field was not selected)

1141 return typing.cast(T_Value, None) # cast as T_Value so mypy understands it for selected fields

1142 else:

1143 # getting as class -> return actual field so pydal understands it when using in query etc.

1144 return typing.cast(TypedField[T_Value], self._field) # pretend it's still typed for IDE support

1145

1146 def __str__(self) -> str:

1147 """

1148 String representation of a Typed Field.

1149

1150 If `type` is set explicitly (e.g. TypedField(str, type="text")), that type is used: `TypedField.text`,

1151 otherwise the type annotation is used (e.g. TypedField(str) -> TypedField.str)

1152 """

1153 return str(self._field) if self._field else ""

1154

1155 def __repr__(self) -> str:

1156 """

1157 More detailed string representation of a Typed Field.

1158

1159 Uses __str__ and adds the provided extra options (kwargs) in the representation.

1160 """

1161 s = self.__str__()

1162

1163 if "type" in self.kwargs:

1164 # manual type in kwargs supplied

1165 t = self.kwargs["type"]

1166 elif issubclass(type, type(self._type)):

1167 # normal type, str.__name__ = 'str'

1168 t = getattr(self._type, "__name__", str(self._type))

1169 elif t_args := typing.get_args(self._type):

1170 # list[str] -> 'str'

1171 t = t_args[0].__name__

1172 else: # pragma: no cover

1173 # fallback - something else, may not even happen, I'm not sure

1174 t = self._type

1175

1176 s = f"TypedField[{t}].{s}" if s else f"TypedField[{t}]"

1177

1178 kw = self.kwargs.copy()

1179 kw.pop("type", None)

1180 return f"<{s} with options {kw}>"

1181

1182 def _to_field(self, extra_kwargs: typing.MutableMapping[str, Any]) -> Optional[str]:

1183 """

1184 Convert a Typed Field instance to a pydal.Field.

1185 """

1186 other_kwargs = self.kwargs.copy()

1187 extra_kwargs.update(other_kwargs)

1188 return extra_kwargs.pop("type", False) or TypeDAL._annotation_to_pydal_fieldtype(self._type, extra_kwargs)

1189

1190 def bind(self, field: pydal.objects.Field, table: pydal.objects.Table) -> None:

1191 """

1192 Bind the right db/table/field info to this class, so queries can be made using `Class.field == ...`.

1193 """

1194 self._table = table

1195 self._field = field

1196

1197 def __getattr__(self, key: str) -> Any:

1198 """

1199 If the regular getattribute does not work, try to get info from the related Field.

1200 """

1201 with contextlib.suppress(AttributeError):

1202 return super().__getattribute__(key)

1203

1204 # try on actual field:

1205 return getattr(self._field, key)

1206

1207 def __eq__(self, other: Any) -> Query:

1208 """

1209 Performing == on a Field will result in a Query.

1210 """

1211 return typing.cast(Query, self._field == other)

1212

1213 def __ne__(self, other: Any) -> Query:

1214 """

1215 Performing != on a Field will result in a Query.

1216 """

1217 return typing.cast(Query, self._field != other)

1218

1219 def __gt__(self, other: Any) -> Query:

1220 """

1221 Performing > on a Field will result in a Query.

1222 """

1223 return typing.cast(Query, self._field > other)

1224

1225 def __lt__(self, other: Any) -> Query:

1226 """

1227 Performing < on a Field will result in a Query.

1228 """

1229 return typing.cast(Query, self._field < other)

1230

1231 def __ge__(self, other: Any) -> Query:

1232 """

1233 Performing >= on a Field will result in a Query.

1234 """

1235 return typing.cast(Query, self._field >= other)

1236

1237 def __le__(self, other: Any) -> Query:

1238 """

1239 Performing <= on a Field will result in a Query.

1240 """

1241 return typing.cast(Query, self._field <= other)

1242

1243 def __hash__(self) -> int:

1244 """

1245 Shadow Field.__hash__.

1246 """

1247 return hash(self._field)

1248

1249 def __invert__(self) -> Expression:

1250 """

1251 Performing ~ on a Field will result in an Expression.

1252 """

1253 if not self._field: # pragma: no cover

1254 raise ValueError("Unbound Field can not be inverted!")

1255

1256 return typing.cast(Expression, ~self._field)

1257

1258

1259 class TypedTable(metaclass=TableMeta):

1260 """

1261 Enhanded modeling system on top of pydal's Table that adds typing and additional functionality.

1262 """

1263

1264 # set up by 'new':

1265 _row: Row | None = None

1266

1267 _with: list[str]

1268

1269 id: "TypedField[int]" # noqa: A003

1270

1271 def _setup_instance_methods(self) -> None:

1272 self.as_dict = self._as_dict # type: ignore

1273 self.__json__ = self.as_json = self._as_json # type: ignore

1274 # self.as_yaml = self._as_yaml # type: ignore

1275 self.as_xml = self._as_xml # type: ignore

1276

1277 self.update = self._update # type: ignore

1278

1279 self.delete_record = self._delete_record # type: ignore

1280 self.update_record = self._update_record # type: ignore

1281

1282 def __new__(

1283 cls, row_or_id: typing.Union[Row, Query, pydal.objects.Set, int, str, None, "TypedTable"] = None, **filters: Any

1284 ) -> "TypedTable":

1285 """

1286 Create a Typed Rows model instance from an existing row, ID or query.

1287

1288 Examples:

1289 MyTable(1)

1290 MyTable(id=1)

1291 MyTable(MyTable.id == 1)

1292 """

1293 table = cls._ensure_table_defined()

1294 inst = super().__new__(cls)

1295

1296 if isinstance(row_or_id, TypedTable):

1297 # existing typed table instance!

1298 return row_or_id

1299 elif isinstance(row_or_id, pydal.objects.Row):

1300 row = row_or_id

1301 elif row_or_id is not None:

1302 row = table(row_or_id, **filters)

1303 elif filters:

1304 row = table(**filters)

1305 else:

1306 # dummy object

1307 return inst

1308

1309 if not row:

1310 return None # type: ignore

1311

1312 inst._row = row

1313 inst.__dict__.update(row)

1314 inst._setup_instance_methods()

1315 return inst

1316

1317 def __iter__(self) -> typing.Generator[Any, None, None]:

1318 """

1319 Allows looping through the columns.

1320 """

1321 row = self._ensure_matching_row()

1322 yield from iter(row)

1323

1324 def __getitem__(self, item: str) -> Any:

1325 """

1326 Allows dictionary notation to get columns.

1327 """

1328 if item in self.__dict__:

1329 return self.__dict__.get(item)

1330

1331 # fallback to lookup in row

1332 if self._row:

1333 return self._row[item]

1334

1335 # nothing found!

1336 raise KeyError(item)

1337

1338 def __getattr__(self, item: str) -> Any:

1339 """

1340 Allows dot notation to get columns.

1341 """

1342 if value := self.get(item):

1343 return value

1344

1345 raise AttributeError(item)

1346

1347 def get(self, item: str, default: Any = None) -> Any:

1348 """

1349 Try to get a column from this instance, else return default.

1350 """

1351 try:

1352 return self.__getitem__(item)

1353 except KeyError:

1354 return default

1355

1356 def __setitem__(self, key: str, value: Any) -> None:

1357 """

1358 Data can both be updated via dot and dict notation.

1359 """

1360 return setattr(self, key, value)

1361

1362 def __int__(self) -> int:

1363 """

1364 Calling int on a model instance will return its id.

1365 """

1366 return getattr(self, "id", 0)

1367

1368 def __bool__(self) -> bool:

1369 """

1370 If the instance has an underlying row with data, it is truthy.

1371 """

1372 return bool(getattr(self, "_row", False))

1373

1374 def _ensure_matching_row(self) -> Row:

1375 if not getattr(self, "_row", None):

1376 raise EnvironmentError("Trying to access non-existant row. Maybe it was deleted or not yet initialized?")

1377 return self._row

1378

1379 def __repr__(self) -> str:

1380 """

1381 String representation of the model instance.

1382 """

1383 model_name = self.__class__.__name__

1384 model_data = {}

1385

1386 if self._row:

1387 model_data = self._row.as_json()

1388

1389 details = model_name

1390 details += f"({model_data})"

1391

1392 if relationships := getattr(self, "_with", []):

1393 details += f" + {relationships}"

1394

1395 return f"<{details}>"

1396

1397 # serialization

1398 # underscore variants work for class instances (set up by _setup_instance_methods)

1399

1400 @classmethod

1401 def as_dict(cls, flat: bool = False, sanitize: bool = True) -> dict[str, Any]:

1402 """

1403 Dump the object to a plain dict.

1404

1405 Can be used as both a class or instance method:

1406 - dumps the table info if it's a class

1407 - dumps the row info if it's an instance (see _as_dict)

1408 """

1409 table = cls._ensure_table_defined()

1410 result = table.as_dict(flat, sanitize)

1411 return typing.cast(dict[str, Any], result)

1412

1413 @classmethod

1414 def as_json(cls, sanitize: bool = True) -> str:

1415 """

1416 Dump the object to json.

1417

1418 Can be used as both a class or instance method:

1419 - dumps the table info if it's a class

1420 - dumps the row info if it's an instance (see _as_json)

1421 """

1422 table = cls._ensure_table_defined()

1423 return typing.cast(str, table.as_json(sanitize))

1424

1425 @classmethod

1426 def as_xml(cls, sanitize: bool = True) -> str: # pragma: no cover

1427 """

1428 Dump the object to xml.

1429

1430 Can be used as both a class or instance method:

1431 - dumps the table info if it's a class

1432 - dumps the row info if it's an instance (see _as_xml)

1433 """

1434 table = cls._ensure_table_defined()

1435 return typing.cast(str, table.as_xml(sanitize))

1436

1437 @classmethod

1438 def as_yaml(cls, sanitize: bool = True) -> str:

1439 """

1440 Dump the object to yaml.

1441

1442 Can be used as both a class or instance method:

1443 - dumps the table info if it's a class

1444 - dumps the row info if it's an instance (see _as_yaml)

1445 """

1446 table = cls._ensure_table_defined()

1447 return typing.cast(str, table.as_yaml(sanitize))

1448

1449 def _as_dict(

1450 self, datetime_to_str: bool = False, custom_types: typing.Iterable[type] | type | None = None

1451 ) -> dict[str, Any]:

1452 row = self._ensure_matching_row()

1453 result = row.as_dict(datetime_to_str=datetime_to_str, custom_types=custom_types)

1454

1455 if _with := getattr(self, "_with", None):

1456 for relationship in _with:

1457 data = self.get(relationship)

1458 if isinstance(data, list):

1459 data = [_.as_dict() if getattr(_, "as_dict", None) else _ for _ in data]

1460 elif data:

1461 data = data.as_dict()

1462

1463 result[relationship] = data

1464

1465 return typing.cast(dict[str, Any], result)

1466

1467 def _as_json(

1468 self,

1469 mode: str = "object",

1470 default: typing.Callable[[Any], Any] = None,

1471 colnames: list[str] = None,

1472 serialize: bool = True,

1473 **kwargs: Any,

1474 ) -> str:

1475 row = self._ensure_matching_row()

1476 return typing.cast(str, row.as_json(mode, default, colnames, serialize, *kwargs))

1477

1478 def _as_xml(self, sanitize: bool = True) -> str: # pragma: no cover

1479 row = self._ensure_matching_row()

1480 return typing.cast(str, row.as_xml(sanitize))

1481

1482 # def _as_yaml(self, sanitize: bool = True) -> str:

1483 # row = self._ensure_matching_row()

1484 # return typing.cast(str, row.as_yaml(sanitize))

1485

1486 def __setattr__(self, key: str, value: Any) -> None:

1487 """

1488 When setting a property on a Typed Table model instance, also update the underlying row.

1489 """

1490 if self._row and key in self._row.__dict__ and not callable(value):

1491 # enables `row.key = value; row.update_record()`

1492 self._row[key] = value

1493

1494 super().__setattr__(key, value)

1495

1496 @classmethod

1497 def update(cls: typing.Type[T_MetaInstance], query: Query, **fields: Any) -> T_MetaInstance | None:

1498 """

1499 Update one record.

1500

1501 Example:

1502 MyTable.update(MyTable.id == 1, name="NewName") -> MyTable

1503 """

1504 # todo: update multiple?

1505 if record := cls(query):

1506 return record.update_record(**fields)

1507 else:

1508 return None

1509

1510 def _update(self: T_MetaInstance, **fields: Any) -> T_MetaInstance:

1511 row = self._ensure_matching_row()

1512 row.update(**fields)

1513 self.__dict__.update(**fields)

1514 return self

1515

1516 def _update_record(self: T_MetaInstance, **fields: Any) -> T_MetaInstance:

1517 row = self._ensure_matching_row()

1518 new_row = row.update_record(**fields)

1519 self.update(**new_row)

1520 return self

1521

1522 def update_record(self: T_MetaInstance, **fields: Any) -> T_MetaInstance: # pragma: no cover

1523 """

1524 Here as a placeholder for _update_record.

1525

1526 Will be replaced on instance creation!

1527 """

1528 return self._update_record(**fields)

1529

1530 def _delete_record(self) -> int:

1531 """

1532 Actual logic in `pydal.helpers.classes.RecordDeleter`.

1533 """

1534 row = self._ensure_matching_row()

1535 result = row.delete_record()

1536 self.__dict__ = {} # empty self, since row is no more.

1537 self._row = None # just to be sure

1538 self._setup_instance_methods()

1539 # ^ instance methods might've been deleted by emptying dict,

1540 # but we still want .as_dict to show an error, not the table's as_dict.

1541 return typing.cast(int, result)

1542

1543 def delete_record(self) -> int: # pragma: no cover

1544 """

1545 Here as a placeholder for _delete_record.

1546

1547 Will be replaced on instance creation!

1548 """

1549 return self._delete_record()

1550

1551 # __del__ is also called on the end of a scope so don't remove records on every del!!

1552

1553 # pickling:

1554 def __setstate__(self, state: dict[str, Any]) -> None:

1555 """

1556 Used by dill when loading from a bytestring.

1557 """

1558 # as_dict also includes table info, so dump as json to only get the actual row data

1559 # then create a new (more empty) row object:

1560 state["_row"] = Row(json.loads(state["_row"]))

1561 self.__dict__ |= state

1562

1563 def __getstate__(self) -> dict[str, Any]:

1564 """

1565 State to save when pickling.

1566

1567 Prevents db connection from being pickled.

1568 Similar to as_dict but without changing the data of the relationships (dill does that recursively)

1569 """

1570 row = self._ensure_matching_row()

1571 result: dict[str, Any] = row.as_dict()

1572

1573 if _with := getattr(self, "_with", None):

1574 for relationship in _with:

1575 data = self.get(relationship)

1576

1577 result[relationship] = data

1578

1579 result["_row"] = self._row.as_json() if self._row else ""

1580 return result

1581

1582

1583# backwards compat:

1584 TypedRow = TypedTable

1585

1586

1587 class TypedRows(typing.Collection[T_MetaInstance], Rows):

1588 """

1589 Slighly enhaned and typed functionality on top of pydal Rows (the result of a select).

1590 """

1591

1592 records: dict[int, T_MetaInstance]

1593 # _rows: Rows

1594 model: typing.Type[T_MetaInstance]

1595 metadata: Metadata

1596

1597 # pseudo-properties: actually stored in _rows

1598 db: TypeDAL

1599 colnames: list[str]

1600 fields: list[Field]

1601 colnames_fields: list[Field]

1602 response: list[tuple[Any, ...]]

1603

1604 def __init__(

1605 self,

1606 rows: Rows,

1607 model: typing.Type[T_MetaInstance],

1608 records: dict[int, T_MetaInstance] = None,

1609 metadata: Metadata = None,

1610 ) -> None:

1611 """

1612 Should not be called manually!

1613

1614 Normally, the `records` from an existing `Rows` object are used

1615 but these can be overwritten with a `records` dict.

1616 `metadata` can be any (un)structured data

1617 `model` is a Typed Table class

1618 """

1619 records = records or {row.id: model(row) for row in rows}

1620 super().__init__(rows.db, records, rows.colnames, rows.compact, rows.response, rows.fields)

1621 self.model = model

1622 self.metadata = metadata or {}

1623

1624 def __len__(self) -> int:

1625 """

1626 Return the count of rows.

1627 """

1628 return len(self.records)

1629

1630 def __iter__(self) -> typing.Iterator[T_MetaInstance]:

1631 """

1632 Loop through the rows.

1633 """

1634 yield from self.records.values()

1635

1636 def __contains__(self, ind: Any) -> bool:

1637 """

1638 Check if an id exists in this result set.

1639 """

1640 return ind in self.records

1641

1642 def first(self) -> T_MetaInstance | None:

1643 """

1644 Get the row with the lowest id.

1645 """

1646 if not self.records:

1647 return None

1648

1649 return next(iter(self))

1650

1651 def last(self) -> T_MetaInstance | None:

1652 """

1653 Get the row with the highest id.

1654 """

1655 if not self.records:

1656 return None

1657

1658 max_id = max(self.records.keys())

1659 return self[max_id]

1660

1661 def find(

1662 self, f: typing.Callable[[T_MetaInstance], Query], limitby: tuple[int, int] = None

1663 ) -> "TypedRows[T_MetaInstance]":

1664 """

1665 Returns a new Rows object, a subset of the original object, filtered by the function `f`.

1666 """

1667 if not self.records:

1668 return self.__class__(self, self.model, {})

1669

1670 records = {}

1671 if limitby:

1672 _min, _max = limitby

1673 else:

1674 _min, _max = 0, len(self)

1675 count = 0

1676 for i, row in self.records.items():

1677 if f(row):

1678 if _min <= count:

1679 records[i] = row

1680 count += 1

1681 if count == _max:

1682 break

1683

1684 return self.__class__(self, self.model, records)

1685

1686 def exclude(self, f: typing.Callable[[T_MetaInstance], Query]) -> "TypedRows[T_MetaInstance]":

1687 """

1688 Removes elements from the calling Rows object, filtered by the function `f`, \

1689 and returns a new Rows object containing the removed elements.

1690 """

1691 if not self.records:

1692 return self.__class__(self, self.model, {})

1693 removed = {}

1694 to_remove = []

1695 for i in self.records:

1696 row = self[i]

1697 if f(row):

1698 removed[i] = self.records[i]

1699 to_remove.append(i)

1700

1701 [self.records.pop(i) for i in to_remove]

1702

1703 return self.__class__(

1704 self,

1705 self.model,

1706 removed,

1707 )

1708

1709 def sort(self, f: typing.Callable[[T_MetaInstance], Any], reverse: bool = False) -> list[T_MetaInstance]:

1710 """

1711 Returns a list of sorted elements (not sorted in place).

1712 """

1713 return [r for (r, s) in sorted(zip(self.records.values(), self), key=lambda r: f(r[1]), reverse=reverse)]

1714

1715 def __str__(self) -> str:

1716 """

1717 Simple string representation.

1718 """

1719 return f"<TypedRows with {len(self)} records>"

1720

1721 def __repr__(self) -> str:

1722 """

1723 Print a table on repr().

1724 """

1725 data = self.as_dict()

1726 headers = list(next(iter(data.values())).keys())

1727 return mktable(data, headers)

1728

1729 def group_by_value(

1730 self, *fields: "str | Field | TypedField[T]", one_result: bool = False, **kwargs: Any

1731 ) -> dict[T, list[T_MetaInstance]]:

1732 """

1733 Group the rows by a specific field (which will be the dict key).

1734 """

1735 kwargs["one_result"] = one_result

1736 result = super().group_by_value(*fields, **kwargs)

1737 return typing.cast(dict[T, list[T_MetaInstance]], result)

1738

1739 def column(self, column: str = None) -> list[Any]:

1740 """

1741 Get a list of all values in a specific column.

1742

1743 Example:

1744 rows.column('name') -> ['Name 1', 'Name 2', ...]

1745 """

1746 return typing.cast(list[Any], super().column(column))

1747

1748 def as_csv(self) -> str:

1749 """

1750 Dump the data to csv.

1751 """

1752 return typing.cast(str, super().as_csv())

1753

1754 def as_dict(

1755 self,

1756 key: str = None,

1757 compact: bool = False,

1758 storage_to_dict: bool = False,

1759 datetime_to_str: bool = False,

1760 custom_types: list[type] = None,

1761 ) -> dict[int, dict[str, Any]]:

1762 """

1763 Get the data in a dict of dicts.

1764 """

1765 if any([key, compact, storage_to_dict, datetime_to_str, custom_types]):

1766 # functionality not guaranteed

1767 return typing.cast(

1768 dict[int, dict[str, Any]],

1769 super().as_dict(

1770 key or "id",

1771 compact,

1772 storage_to_dict,

1773 datetime_to_str,

1774 custom_types,

1775 ),

1776 )

1777

1778 return {k: v.as_dict() for k, v in self.records.items()}

1779

1780 def as_json(self, mode: str = "object", default: typing.Callable[[Any], Any] = None) -> str:

1781 """

1782 Turn the data into a dict and then dump to JSON.

1783 """

1784 return typing.cast(str, super().as_json(mode=mode, default=default))

1785

1786 def json(self, mode: str = "object", default: typing.Callable[[Any], Any] = None) -> str:

1787 """

1788 Turn the data into a dict and then dump to JSON.

1789 """

1790 return typing.cast(str, super().as_json(mode=mode, default=default))

1791

1792 def as_list(

1793 self,

1794 compact: bool = False,

1795 storage_to_dict: bool = False,

1796 datetime_to_str: bool = False,

1797 custom_types: list[type] = None,

1798 ) -> list[dict[str, Any]]:

1799 """

1800 Get the data in a list of dicts.

1801 """

1802 if any([compact, storage_to_dict, datetime_to_str, custom_types]):

1803 return typing.cast(

1804 list[dict[str, Any]], super().as_list(compact, storage_to_dict, datetime_to_str, custom_types)

1805 )

1806 return [_.as_dict() for _ in self.records.values()]

1807

1808 def __getitem__(self, item: int) -> T_MetaInstance:

1809 """

1810 You can get a specific row by ID from a typedrows by using rows[idx] notation.

1811

1812 Since pydal's implementation differs (they expect a list instead of a dict with id keys),

1813 using rows[0] will return the first row, regardless of its id.

1814 """

1815 try:

1816 return self.records[item]

1817 except KeyError as e:

1818 if item == 0 and (row := self.first()):

1819 # special case: pydal internals think Rows.records is a list, not a dict

1820 return row

1821

1822 raise e

1823

1824 def get(self, item: int) -> typing.Optional[T_MetaInstance]:

1825 """

1826 Get a row by ID, or receive None if it isn't in this result set.

1827 """

1828 return self.records.get(item)

1829

1830 def join(

1831 self,

1832 field: "Field | TypedField[Any]",

1833 name: str = None,

1834 constraint: Query = None,

1835 fields: list[str | Field] = None,

1836 orderby: Optional[str | Field] = None,

1837 ) -> T_MetaInstance:

1838 """

1839 This can be used to JOIN with some relationships after the initial select.

1840

1841 Using the querybuilder's .join() method is prefered!

1842 """

1843 result = super().join(field, name, constraint, fields or [], orderby)

1844 return typing.cast(T_MetaInstance, result)

1845

1846 def export_to_csv_file(

1847 self,

1848 ofile: typing.TextIO,

1849 null: Any = "<NULL>",

1850 delimiter: str = ",",

1851 quotechar: str = '"',

1852 quoting: int = csv.QUOTE_MINIMAL,

1853 represent: bool = False,

1854 colnames: list[str] = None,

1855 write_colnames: bool = True,

1856 *args: Any,

1857 **kwargs: Any,

1858 ) -> None:

1859 """

1860 Shadow export_to_csv_file from Rows, but with typing.

1861

1862 See http://web2py.com/books/default/chapter/29/06/the-database-abstraction-layer?search=export_to_csv_file#Exporting-and-importing-data

1863 """

1864 super().export_to_csv_file(

1865 ofile,

1866 null,

1867 *args,

1868 delimiter=delimiter,

1869 quotechar=quotechar,

1870 quoting=quoting,

1871 represent=represent,

1872 colnames=colnames or self.colnames,

1873 write_colnames=write_colnames,

1874 **kwargs,

1875 )

1876

1877 @classmethod

1878 def from_rows(

1879 cls, rows: Rows, model: typing.Type[T_MetaInstance], metadata: Metadata = None

1880 ) -> "TypedRows[T_MetaInstance]":

1881 """

1882 Internal method to convert a Rows object to a TypedRows.

1883 """

1884 return cls(rows, model, metadata=metadata)

1885

1886 def __json__(self) -> dict[str, Any]:

1887 """

1888 For json-fix.

1889 """

1890 return typing.cast(dict[str, Any], self.as_dict())

1891

1892 def __getstate__(self) -> dict[str, Any]:

1893 """

1894 Used by dill to dump to bytes (exclude db connection etc).

1895 """

1896 return {

1897 "metadata": json.dumps(self.metadata, default=str),

1898 "records": self.records,

1899 }

1900

1901 def __setstate__(self, state: dict[str, Any]) -> None:

1902 """

1903 Used by dill when loading from a bytestring.

1904 """

1905 state["metadata"] = json.loads(state["metadata"])

1906 self.__dict__.update(state)

1907

1908

1909 from .caching import ( # noqa: E402

1910 _remove_cache,

1911 _TypedalCache,

1912 _TypedalCacheDependency,

1913 create_and_hash_cache_key,

1914 get_expire,

1915 load_from_cache,

1916 save_to_cache,

1917)

1918

1919

1920 class QueryBuilder(typing.Generic[T_MetaInstance]):

1921 """

1922 Abstration on top of pydal's query system.

1923 """

1924

1925 model: typing.Type[T_MetaInstance]

1926 query: Query

1927 select_args: list[Any]

1928 select_kwargs: dict[str, Any]

1929 relationships: dict[str, Relationship[Any]]

1930 metadata: Metadata

1931

1932 def __init__(

1933 self,

1934 model: typing.Type[T_MetaInstance],

1935 add_query: Optional[Query] = None,

1936 select_args: Optional[list[Any]] = None,

1937 select_kwargs: Optional[dict[str, Any]] = None,

1938 relationships: dict[str, Relationship[Any]] = None,

1939 metadata: Metadata = None,

1940 ):

1941 """

1942 Normally, you wouldn't manually initialize a QueryBuilder but start using a method on a TypedTable.

1943

1944 Example:

1945 MyTable.where(...) -> QueryBuilder[MyTable]

1946 """

1947 self.model = model

1948 table = model._ensure_table_defined()

1949 default_query = typing.cast(Query, table.id > 0)

1950 self.query = add_query or default_query

1951 self.select_args = select_args or []

1952 self.select_kwargs = select_kwargs or {}

1953 self.relationships = relationships or {}

1954 self.metadata = metadata or {}

1955

1956 def __str__(self) -> str:

1957 """

1958 Simple string representation for the query builder.

1959 """

1960 return f"QueryBuilder for {self.model}"

1961

1962 def __repr__(self) -> str:

1963 """

1964 Advanced string representation for the query builder.

1965 """

1966 return (

1967 f"<QueryBuilder for {self.model} with "

1968 f"{len(self.select_args)} select args; "

1969 f"{len(self.select_kwargs)} select kwargs; "

1970 f"{len(self.relationships)} relationships; "

1971 f"query: {bool(self.query)}; "

1972 f"metadata: {self.metadata}; "

1973 f">"

1974 )

1975

1976 def __bool__(self) -> bool:

1977 """

1978 Querybuilder is truthy if it has rows.

1979 """

1980 return self.count() > 0

1981

1982 def _extend(

1983 self,

1984 add_query: Optional[Query] = None,

1985 overwrite_query: Optional[Query] = None,

1986 select_args: Optional[list[Any]] = None,

1987 select_kwargs: Optional[dict[str, Any]] = None,

1988 relationships: dict[str, Relationship[Any]] = None,

1989 metadata: Metadata = None,

1990 ) -> "QueryBuilder[T_MetaInstance]":

1991 return QueryBuilder(

1992 self.model,

1993 (add_query & self.query) if add_query else overwrite_query or self.query,

1994 (self.select_args + select_args) if select_args else self.select_args,

1995 (self.select_kwargs | select_kwargs) if select_kwargs else self.select_kwargs,

1996 (self.relationships | relationships) if relationships else self.relationships,

1997 (self.metadata | (metadata or {})) if metadata else self.metadata,

1998 )

1999

2000 def select(self, *fields: Any, **options: Any) -> "QueryBuilder[T_MetaInstance]":

2001 """

2002 Fields: database columns by name ('id'), by field reference (table.id) or other (e.g. table.ALL).

2003

2004 Options:

2005 paraphrased from the web2py pydal docs,

2006 For more info, see http://www.web2py.com/books/default/chapter/29/06/the-database-abstraction-layer#orderby-groupby-limitby-distinct-having-orderby_on_limitby-join-left-cache

2007

2008 orderby: field(s) to order by. Supported:

2009 table.name - sort by name, ascending

2010 ~table.name - sort by name, descending

2011 <random> - sort randomly

2012 table.name|table.id - sort by two fields (first name, then id)

2013

2014 groupby, having: together with orderby:

2015 groupby can be a field (e.g. table.name) to group records by

2016 having can be a query, only those `having` the condition are grouped

2017

2018 limitby: tuple of min and max. When using the query builder, .paginate(limit, page) is recommended.

2019 distinct: bool/field. Only select rows that differ

2020 orderby_on_limitby (bool, default: True): by default, an implicit orderby is added when doing limitby.

2021 join: othertable.on(query) - do an INNER JOIN. Using TypeDAL relationships with .join() is recommended!

2022 left: othertable.on(query) - do a LEFT JOIN. Using TypeDAL relationships with .join() is recommended!

2023 cache: cache the query result to speed up repeated queries; e.g. (cache=(cache.ram, 3600), cacheable=True)

2024 """

2025 return self._extend(select_args=list(fields), select_kwargs=options)

2026

2027 def where(

2028 self,

2029 *queries_or_lambdas: Query | typing.Callable[[typing.Type[T_MetaInstance]], Query],

2030 **filters: Any,

2031 ) -> "QueryBuilder[T_MetaInstance]":

2032 """

2033 Extend the builder's query.

2034

2035 Can be used in multiple ways:

2036 .where(Query) -> with a direct query such as `Table.id == 5`

2037 .where(lambda table: table.id == 5) -> with a query via a lambda

2038 .where(id=5) -> via keyword arguments

2039

2040 When using multiple where's, they will be ANDed:

2041 .where(lambda table: table.id == 5).where(lambda table: table.id == 6) == (table.id == 5) & (table.id=6)

2042 When passing multiple queries to a single .where, they will be ORed:

2043 .where(lambda table: table.id == 5, lambda table: table.id == 6) == (table.id == 5) | (table.id=6)

2044 """

2045 new_query = self.query

2046 table = self.model._ensure_table_defined()

2047

2048 for field, value in filters.items():

2049 new_query &= table[field] == value

2050

2051 subquery: DummyQuery | Query = DummyQuery()

2052 for query_or_lambda in queries_or_lambdas:

2053 if isinstance(query_or_lambda, _Query):

2054 subquery |= typing.cast(Query, query_or_lambda)

2055 elif callable(query_or_lambda):

2056 if result := query_or_lambda(self.model):

2057 subquery |= result

2058 elif isinstance(query_or_lambda, (Field, _Field)) or is_typed_field(query_or_lambda):

2059 subquery |= typing.cast(Query, query_or_lambda != None)

2060 else:

2061 raise ValueError(f"Unexpected query type ({type(query_or_lambda)}).")

2062

2063 if subquery:

2064 new_query &= subquery

2065

2066 return self._extend(overwrite_query=new_query)

2067

2068 def join(

2069 self,

2070 *fields: str | typing.Type[TypedTable],

2071 method: JOIN_OPTIONS = None,

2072 on: OnQuery | list[Expression] | Expression = None,

2073 condition: Condition = None,

2074 ) -> "QueryBuilder[T_MetaInstance]":

2075 """

2076 Include relationship fields in the result.

2077

2078 `fields` can be names of Relationships on the current model.

2079 If no fields are passed, all will be used.

2080

2081 By default, the `method` defined in the relationship is used.

2082 This can be overwritten with the `method` keyword argument (left or inner)

2083 """

2084 # todo: allow limiting amount of related rows returned for join?

2085

2086 relationships = self.model.get_relationships()

2087

2088 if condition and on:

2089 raise ValueError("condition and on can not be used together!")

2090 elif condition:

2091 if len(fields) != 1:

2092 raise ValueError("join(field, condition=...) can only be used with exactly one field!")

2093

2094 if isinstance(condition, pydal.objects.Query):

2095 condition = as_lambda(condition)

2096

2097 relationships = {str(fields[0]): relationship(fields[0], condition=condition, join=method)}

2098 elif on:

2099 if len(fields) != 1:

2100 raise ValueError("join(field, on=...) can only be used with exactly one field!")

2101

2102 if isinstance(on, pydal.objects.Expression):

2103 on = [on]

2104

2105 if isinstance(on, list):

2106 on = as_lambda(on)

2107 relationships = {str(fields[0]): relationship(fields[0], on=on, join=method)}

2108

2109 else:

2110 if fields:

2111 # join on every relationship

2112 relationships = {str(k): relationships[str(k)] for k in fields}

2113

2114 if method:

2115 relationships = {str(k): r.clone(join=method) for k, r in relationships.items()}

2116

2117 return self._extend(relationships=relationships)

2118

2119 def cache(

2120 self, *deps: Any, expires_at: Optional[dt.datetime] = None, ttl: Optional[int | dt.timedelta] = None

2121 ) -> "QueryBuilder[T_MetaInstance]":

2122 """

2123 Enable caching for this query to load repeated calls from a dill row \

2124 instead of executing the sql and collecing matching rows again.

2125 """

2126 existing = self.metadata.get("cache", {})

2127

2128 metadata: Metadata = {}

2129

2130 cache_meta = typing.cast(

2131 CacheMetadata,

2132 self.metadata.get("cache", {})

2133 | {

2134 "enabled": True,

2135 "depends_on": existing.get("depends_on", []) + [str(_) for _ in deps],

2136 "expires_at": get_expire(expires_at=expires_at, ttl=ttl),

2137 },

2138 )

2139

2140 metadata["cache"] = cache_meta

2141 return self._extend(metadata=metadata)

2142

2143 def _get_db(self) -> TypeDAL:

2144 if db := self.model._db:

2145 return db

2146 else: # pragma: no cover

2147 raise EnvironmentError("@define or db.define is not called on this class yet!")

2148

2149 def _select_arg_convert(self, arg: Any) -> Any:

2150 # typedfield are not really used at runtime anymore, but leave it in for safety:

2151 if isinstance(arg, TypedField): # pragma: no cover

2152 arg = arg._field

2153

2154 return arg

2155

2156 def delete(self) -> list[int]:

2157 """

2158 Based on the current query, delete rows and return a list of deleted IDs.

2159 """

2160 db = self._get_db()

2161 removed_ids = [_.id for _ in db(self.query).select("id")]

2162 if db(self.query).delete():

2163 # success!

2164 return removed_ids

2165

2166 return []

2167

2168 def _delete(self) -> str:

2169 db = self._get_db()

2170 return str(db(self.query)._delete())

2171

2172 def update(self, **fields: Any) -> list[int]:

2173 """

2174 Based on the current query, update `fields` and return a list of updated IDs.

2175 """

2176 # todo: limit?

2177 db = self._get_db()

2178 updated_ids = db(self.query).select("id").column("id")

2179 if db(self.query).update(**fields):

2180 # success!

2181 return updated_ids

2182

2183 return []

2184

2185 def _update(self, **fields: Any) -> str:

2186 db = self._get_db()

2187 return str(db(self.query)._update(**fields))

2188

2189 def _before_query(self, mut_metadata: Metadata, add_id: bool = True) -> tuple[Query, list[Any], dict[str, Any]]:

2190 select_args = [self._select_arg_convert(_) for _ in self.select_args] or [self.model.ALL]

2191 select_kwargs = self.select_kwargs.copy()

2192 query = self.query

2193 model = self.model

2194 mut_metadata["query"] = query

2195 # require at least id of main table:

2196 select_fields = ", ".join([str(_) for _ in select_args])

2197 tablename = str(model)

2198

2199 if add_id and f"{tablename}.id" not in select_fields:

2200 # fields of other selected, but required ID is missing.

2201 select_args.append(model.id)

2202

2203 if self.relationships:

2204 query, select_args = self._handle_relationships_pre_select(query, select_args, select_kwargs, mut_metadata)

2205

2206 return query, select_args, select_kwargs

2207

2208 def to_sql(self, add_id: bool = False) -> str:

2209 """

2210 Generate the SQL for the built query.

2211 """

2212 db = self._get_db()

2213

2214 query, select_args, select_kwargs = self._before_query({}, add_id=add_id)

2215

2216 return str(db(query)._select(*select_args, **select_kwargs))

2217

2218 def _collect(self) -> str:

2219 """

2220 Alias for to_sql, pydal-like syntax.

2221 """

2222 return self.to_sql()

2223

2224 def _collect_cached(self, metadata: Metadata) -> "TypedRows[T_MetaInstance] | None":

2225 expires_at = metadata["cache"].get("expires_at")

2226 metadata["cache"] |= {

2227 # key is partly dependant on cache metadata but not these:

2228 "key": None,

2229 "status": None,

2230 "cached_at": None,

2231 "expires_at": None,

2232 } # type: ignore

2233

2234 _, key = create_and_hash_cache_key(

2235 self.model,

2236 metadata,

2237 self.query,

2238 self.select_args,

2239 self.select_kwargs,

2240 self.relationships.keys(),

2241 )

2242

2243 # re-set after creating key:

2244 metadata["cache"]["expires_at"] = expires_at

2245 metadata["cache"]["key"] = key

2246

2247 return load_from_cache(key)

2248

2249 def collect(

2250 self, verbose: bool = False, _to: typing.Type["TypedRows[Any]"] = None, add_id: bool = True

2251 ) -> "TypedRows[T_MetaInstance]":

2252 """

2253 Execute the built query and turn it into model instances, while handling relationships.

2254 """

2255 if _to is None:

2256 _to = TypedRows

2257

2258 db = self._get_db()

2259 metadata = typing.cast(Metadata, self.metadata.copy())

2260

2261 if metadata.get("cache", {}).get("enabled") and (result := self._collect_cached(metadata)):

2262 return result

2263

2264 query, select_args, select_kwargs = self._before_query(metadata, add_id=add_id)

2265

2266 metadata["sql"] = db(query)._select(*select_args, **select_kwargs)

2267

2268 if verbose: # pragma: no cover

2269 print(metadata["sql"])

2270

2271 rows: Rows = db(query).select(*select_args, **select_kwargs)

2272

2273 metadata["final_query"] = str(query)

2274 metadata["final_args"] = [str(_) for _ in select_args]

2275 metadata["final_kwargs"] = select_kwargs

2276

2277 if verbose: # pragma: no cover

2278 print(rows)

2279

2280 if not self.relationships:

2281 # easy

2282 typed_rows = _to.from_rows(rows, self.model, metadata=metadata)

2283

2284 else:

2285 # harder: try to match rows to the belonging objects

2286 # assume structure of {'table': <data>} per row.

2287 # if that's not the case, return default behavior again

2288 typed_rows = self._collect_with_relationships(rows, metadata=metadata, _to=_to)

2289

2290 # only saves if requested in metadata:

2291 return save_to_cache(typed_rows, rows)

2292

2293 def _handle_relationships_pre_select(

2294 self,

2295 query: Query,

2296 select_args: list[Any],

2297 select_kwargs: dict[str, Any],

2298 metadata: Metadata,

2299 ) -> tuple[Query, list[Any]]:

2300 db = self._get_db()

2301 model = self.model

2302

2303 metadata["relationships"] = set(self.relationships.keys())

2304

2305 # query = self._update_query_for_inner(db, model, query)

2306 join = []

2307 for key, relation in self.relationships.items():

2308 if not relation.condition or relation.join != "inner":

2309 continue

2310

2311 other = relation.get_table(db)

2312 other = other.with_alias(f"{key}_{hash(relation)}")

2313 join.append(other.on(relation.condition(model, other)))

2314

2315 if limitby := select_kwargs.pop("limitby", None):

2316 # if limitby + relationships:

2317 # 1. get IDs of main table entries that match 'query'

2318 # 2. change query to .belongs(id)

2319 # 3. add joins etc

2320

2321 kwargs = {"limitby": limitby}

2322

2323 if join:

2324 kwargs["join"] = join

2325

2326 ids = db(query)._select(model.id, **kwargs)

2327 query = model.id.belongs(ids)

2328 metadata["ids"] = ids

2329

2330 if join:

2331 select_kwargs["join"] = join

2332

2333 left = []

2334

2335 for key, relation in self.relationships.items():

2336 other = relation.get_table(db)

2337 method: JOIN_OPTIONS = relation.join or DEFAULT_JOIN_OPTION

2338

2339 select_fields = ", ".join([str(_) for _ in select_args])

2340 pre_alias = str(other)

2341

2342 if f"{other}." not in select_fields:

2343 # no fields of other selected. add .ALL:

2344 select_args.append(other.ALL)

2345 elif f"{other}.id" not in select_fields:

2346 # fields of other selected, but required ID is missing.

2347 select_args.append(other.id)

2348

2349 if relation.on:

2350 # if it has a .on, it's always a left join!

2351 on = relation.on(model, other)

2352 if not isinstance(on, list): # pragma: no cover

2353 on = [on]

2354

2355 left.extend(on)

2356 elif method == "left":

2357 # .on not given, generate it:

2358 other = other.with_alias(f"{key}_{hash(relation)}")

2359 condition = typing.cast(Query, relation.condition(model, other))

2360 left.append(other.on(condition))

2361 else:

2362 # else: inner join (handled earlier)

2363 other = other.with_alias(f"{key}_{hash(relation)}") # only for replace

2364 # other = other.with_alias(f"{key}_{hash(relation)}")

2365 # query &= relation.condition(model, other)

2366

2367 # if no fields of 'other' are included, add other.ALL

2368 # else: only add other.id if missing

2369 select_fields = ", ".join([str(_) for _ in select_args])

2370

2371 post_alias = str(other).split(" AS ")[-1]

2372 if pre_alias != post_alias:

2373 # replace .select's with aliased:

2374 select_fields = select_fields.replace(

2375 f"{pre_alias}.",

2376 f"{post_alias}.",

2377 )

2378

2379 select_args = select_fields.split(", ")

2380

2381 select_kwargs["left"] = left

2382 return query, select_args

2383

2384 def _collect_with_relationships(

2385 self, rows: Rows, metadata: Metadata, _to: typing.Type["TypedRows[Any]"] = None

2386 ) -> "TypedRows[T_MetaInstance]":

2387 """

2388 Transform the raw rows into Typed Table model instances.

2389 """

2390 db = self._get_db()

2391 main_table = self.model._ensure_table_defined()

2392

2393 records = {}

2394 seen_relations: dict[str, set[str]] = defaultdict(set) # main id -> set of col + id for relation

2395

2396 for row in rows:

2397 main = row[main_table]

2398 main_id = main.id

2399

2400 if main_id not in records:

2401 records[main_id] = self.model(main)

2402 records[main_id]._with = list(self.relationships.keys())

2403

2404 # setup up all relationship defaults (once)

2405 for col, relationship in self.relationships.items():

2406 records[main_id][col] = [] if relationship.multiple else None

2407

2408 # now add other relationship data

2409 for column, relation in self.relationships.items():

2410 relationship_column = f"{column}_{hash(relation)}"

2411

2412 # relationship_column works for aliases with the same target column.

2413 # if col + relationship not in the row, just use the regular name.

2414

2415 relation_data = (

2416 row[relationship_column] if relationship_column in row else row[relation.get_table_name()]

2417 )

2418

2419 if relation_data.id is None:

2420 # always skip None ids

2421 continue

2422

2423 if f"{column}-{relation_data.id}" in seen_relations[main_id]:

2424 # speed up duplicates

2425 continue

2426 else:

2427 seen_relations[main_id].add(f"{column}-{relation_data.id}")

2428

2429 relation_table = relation.get_table(db)

2430 # hopefully an instance of a typed table and a regular row otherwise:

2431 instance = relation_table(relation_data) if looks_like(relation_table, TypedTable) else relation_data

2432

2433 if relation.multiple:

2434 # create list of T

2435 if not isinstance(records[main_id].get(column), list): # pragma: no cover

2436 # should already be set up before!

2437 setattr(records[main_id], column, [])

2438

2439 records[main_id][column].append(instance)

2440 else:

2441 # create single T

2442 records[main_id][column] = instance

2443

2444 return _to(rows, self.model, records, metadata=metadata)

2445

2446 def collect_or_fail(self, exception: Exception = None) -> "TypedRows[T_MetaInstance]":

2447 """

2448 Call .collect() and raise an error if nothing found.

2449

2450 Basically unwraps Optional type.

2451 """

2452 if result := self.collect():

2453 return result

2454

2455 if not exception:

2456 exception = ValueError("Nothing found!")

2457

2458 raise exception

2459

2460 def __iter__(self) -> typing.Generator[T_MetaInstance, None, None]:

2461 """

2462 You can start iterating a Query Builder object before calling collect, for ease of use.

2463 """

2464 yield from self.collect()

2465

2466 def count(self) -> int:

2467 """

2468 Return the amount of rows matching the current query.

2469 """

2470 db = self._get_db()

2471 model = self.model

2472 query = self.query

2473

2474 for key, relation in self.relationships.items():

2475 if not relation.condition or relation.join != "inner":

2476 continue

2477

2478 other = relation.get_table(db)

2479 other = other.with_alias(f"{key}_{hash(relation)}")

2480 query &= relation.condition(model, other)

2481

2482 return db(query).count()

2483

2484 def __paginate(

2485 self,

2486 limit: int,

2487 page: int = 1,

2488 ) -> "QueryBuilder[T_MetaInstance]":

2489 _from = limit * (page - 1)

2490 _to = limit * page

2491

2492 available = self.count()

2493

2494 metadata: Metadata = {}

2495

2496 metadata["pagination"] = {

2497 "limit": limit,

2498 "current_page": page,

2499 "max_page": math.ceil(available / limit),

2500 "rows": available,

2501 "min_max": (_from, _to),

2502 }

2503

2504 return self._extend(select_kwargs={"limitby": (_from, _to)}, metadata=metadata)

2505

2506 def paginate(self, limit: int, page: int = 1, verbose: bool = False) -> "PaginatedRows[T_MetaInstance]":

2507 """

2508 Paginate transforms the more readable `page` and `limit` to pydals internal limit and offset.

2509

2510 Note: when using relationships, this limit is only applied to the 'main' table and any number of extra rows \

2511 can be loaded with relationship data!

2512 """

2513 builder = self.__paginate(limit, page)

2514

2515 rows = typing.cast(PaginatedRows[T_MetaInstance], builder.collect(verbose=verbose, _to=PaginatedRows))

2516

2517 rows._query_builder = builder

2518 return rows

2519

2520 def _paginate(

2521 self,

2522 limit: int,

2523 page: int = 1,

2524 ) -> str:

2525 builder = self.__paginate(limit, page)

2526 return builder._collect()

2527

2528 def chunk(self, chunk_size: int) -> typing.Generator["TypedRows[T_MetaInstance]", Any, None]:

2529 """

2530 Generator that yields rows from a paginated source in chunks.

2531

2532 This function retrieves rows from a paginated data source in chunks of the

2533 specified `chunk_size` and yields them as TypedRows.

2534

2535 Example:

2536 ```

2537 for chunk_of_rows in Table.where(SomeTable.id > 5).chunk(100):

2538 for row in chunk_of_rows:

2539 # Process each row within the chunk.

2540 pass

2541 ```

2542 """

2543 page = 1

2544

2545 while rows := self.__paginate(chunk_size, page).collect():

2546 yield rows

2547 page += 1

2548

2549 def first(self, verbose: bool = False) -> T_MetaInstance | None:

2550 """

2551 Get the first row matching the currently built query.

2552

2553 Also adds paginate, since it would be a waste to select more rows than needed.

2554 """

2555 if row := self.paginate(page=1, limit=1, verbose=verbose).first():

2556 return self.model.from_row(row)

2557 else:

2558 return None

2559

2560 def _first(self) -> str:

2561 return self._paginate(page=1, limit=1)

2562

2563 def first_or_fail(self, exception: Exception = None, verbose: bool = False) -> T_MetaInstance:

2564 """

2565 Call .first() and raise an error if nothing found.

2566

2567 Basically unwraps Optional type.

2568 """

2569 if inst := self.first(verbose=verbose):

2570 return inst

2571

2572 if not exception:

2573 exception = ValueError("Nothing found!")

2574

2575 raise exception

2576

2577

2578 S = typing.TypeVar("S")

2579

2580

2581 class PaginatedRows(TypedRows[T_MetaInstance]):

2582 """

2583 Extension on top of rows that is used when calling .paginate() instead of .collect().

2584 """

2585

2586 _query_builder: QueryBuilder[T_MetaInstance]

2587

2588 @property

2589 def data(self) -> list[T_MetaInstance]:

2590 """

2591 Get the underlying data.

2592 """

2593 return list(self.records.values())

2594

2595 @property

2596 def pagination(self) -> Pagination:

2597 """

2598 Get all page info.

2599 """

2600 pagination_data = self.metadata["pagination"]

2601

2602 has_next_page = pagination_data["current_page"] < pagination_data["max_page"]

2603 has_prev_page = pagination_data["current_page"] > 1

2604 return {

2605 "total_items": pagination_data["rows"],

2606 "current_page": pagination_data["current_page"],

2607 "per_page": pagination_data["limit"],

2608 "total_pages": pagination_data["max_page"],

2609 "has_next_page": has_next_page,

2610 "has_prev_page": has_prev_page,

2611 "next_page": pagination_data["current_page"] + 1 if has_next_page else None,

2612 "prev_page": pagination_data["current_page"] - 1 if has_prev_page else None,

2613 }

2614

2615 def next(self) -> Self: # noqa: A003

2616 """

2617 Get the next page.

2618 """

2619 data = self.metadata["pagination"]

2620 if data["current_page"] >= data["max_page"]:

2621 raise StopIteration("Final Page")

2622

2623 return self._query_builder.paginate(limit=data["limit"], page=data["current_page"] + 1)

2624

2625 def previous(self) -> Self:

2626 """

2627 Get the previous page.

2628 """

2629 data = self.metadata["pagination"]

2630 if data["current_page"] <= 1:

2631 raise StopIteration("First Page")

2632

2633 return self._query_builder.paginate(limit=data["limit"], page=data["current_page"] - 1)

2634

2635 def as_dict(self, *_: Any, **__: Any) -> PaginateDict: # type: ignore

2636 """

2637 Convert to a dictionary with pagination info and original data.

2638

2639 All arguments are ignored!

2640 """

2641 return {"data": super().as_dict(), "pagination": self.pagination}

2642

2643

2644 class TypedSet(pydal.objects.Set): # type: ignore # pragma: no cover

2645 """

2646 Used to make pydal Set more typed.

2647

2648 This class is not actually used, only 'cast' by TypeDAL.__call__

2649 """

2650

2651 def count(self, distinct: bool = None, cache: dict[str, Any] = None) -> int:

2652 """

2653 Count returns an int.

2654 """

2655 result = super().count(distinct, cache)

2656 return typing.cast(int, result)

2657

2658 def select(self, *fields: Any, **attributes: Any) -> TypedRows[T_MetaInstance]:

2659 """

2660 Select returns a TypedRows of a user defined table.

2661

2662 Example:

2663 result: TypedRows[MyTable] = db(MyTable.id > 0).select()

2664

2665 for row in result:

2666 typing.reveal_type(row) # MyTable

2667 """

2668 rows = super().select(*fields, **attributes)

2669 return typing.cast(TypedRows[T_MetaInstance], rows)