Coverage for src/typedal/core.py: 99%

1"""

2 Core functionality of TypeDAL.

3"""

4 import contextlib

5 import csv

6 import datetime as dt

7 import inspect

8 import json

9 import math

10 import types

11 import typing

12 import warnings

13 from collections import defaultdict

14 from decimal import Decimal

15 from pathlib import Path

16 from typing import Any, Optional

17

18 import pydal

19 from pydal._globals import DEFAULT

20 from pydal.objects import Field as _Field

21 from pydal.objects import Query as _Query

22 from pydal.objects import Row, Rows

23 from pydal.objects import Table as _Table

24 from typing_extensions import Self

25

26 from .helpers import (

27 DummyQuery,

28 all_annotations,

29 all_dict,

30 as_lambda,

31 extract_type_optional,

32 filter_out,

33 instanciate,

34 is_union,

35 looks_like,

36 mktable,

37 origin_is_subclass,

38 to_snake,

39 unwrap_type,

40)

41 from .types import (

42 AfterDeleteCallable,

43 AfterInsertCallable,

44 AfterUpdateCallable,

45 BeforeDeleteCallable,

46 BeforeInsertCallable,

47 BeforeUpdateCallable,

48 CacheMetadata,

49 Expression,

50 Field,

51 Metadata,

52 PaginateDict,

53 Pagination,

54 Query,

55 _Types,

56)

57

58# use typing.cast(type, ...) to make mypy happy with unions

59 T_annotation = typing.Type[Any] | types.UnionType

60 T_Query = typing.Union["Table", Query, bool, None, "TypedTable", typing.Type["TypedTable"]]

61 T_Value = typing.TypeVar("T_Value") # actual type of the Field (via Generic)

62 T_MetaInstance = typing.TypeVar("T_MetaInstance", bound="TypedTable") # bound="TypedTable"; bound="TableMeta"

63 T = typing.TypeVar("T")

64

65 BASIC_MAPPINGS: dict[T_annotation, str] = {

66 str: "string",

67 int: "integer",

68 bool: "boolean",

69 bytes: "blob",

70 float: "double",

71 object: "json",

72 Decimal: "decimal(10,2)",

73 dt.date: "date",

74 dt.time: "time",

75 dt.datetime: "datetime",

76}

77

78

79 def is_typed_field(cls: Any) -> typing.TypeGuard["TypedField[Any]"]:

80 """

81 Is `cls` an instance or subclass of TypedField?

82

83 Deprecated

84 """

85 return (

86 isinstance(cls, TypedField)

87 or isinstance(typing.get_origin(cls), type)

88 and issubclass(typing.get_origin(cls), TypedField)

89 )

90

91

92 JOIN_OPTIONS = typing.Literal["left", "inner", None]

93 DEFAULT_JOIN_OPTION: JOIN_OPTIONS = "left"

94

95# table-ish paramter:

96 P_Table = typing.Union[typing.Type["TypedTable"], pydal.objects.Table]

97

98 Condition: typing.TypeAlias = typing.Optional[

99 typing.Callable[

100 # self, other -> Query

101 [P_Table, P_Table],

102 Query | bool,

103 ]

104]

105

106 OnQuery: typing.TypeAlias = typing.Optional[

107 typing.Callable[

108 # self, other -> list of .on statements

109 [P_Table, P_Table],

110 list[Expression],

111 ]

112]

113

114 To_Type = typing.TypeVar("To_Type", type[Any], typing.Type[Any], str)

115

116

117 class Relationship(typing.Generic[To_Type]):

118 """

119 Define a relationship to another table.

120 """

121

122 _type: To_Type

123 table: typing.Type["TypedTable"] | type | str

124 condition: Condition

125 on: OnQuery

126 multiple: bool

127 join: JOIN_OPTIONS

128

129 def __init__(

130 self,

131 _type: To_Type,

132 condition: Condition = None,

133 join: JOIN_OPTIONS = None,

134 on: OnQuery = None,

135 ):

136 """

137 Should not be called directly, use relationship() instead!

138 """

139 if condition and on:

140 warnings.warn(f"Relation | Both specified! {condition=} {on=} {_type=}")

141 raise ValueError("Please specify either a condition or an 'on' statement for this relationship!")

142

143 self._type = _type

144 self.condition = condition

145 self.join = "left" if on else join # .on is always left join!

146 self.on = on

147

148 if args := typing.get_args(_type):

149 self.table = unwrap_type(args[0])

150 self.multiple = True

151 else:

152 self.table = _type

153 self.multiple = False

154

155 if isinstance(self.table, str):

156 self.table = TypeDAL.to_snake(self.table)

157

158 def clone(self, **update: Any) -> "Relationship[To_Type]":

159 """

160 Create a copy of the relationship, possibly updated.

161 """

162 return self.__class__(

163 update.get("_type") or self._type,

164 update.get("condition") or self.condition,

165 update.get("join") or self.join,

166 update.get("on") or self.on,

167 )

168

169 def __repr__(self) -> str:

170 """

171 Representation of the relationship.

172 """

173 if callback := self.condition or self.on:

174 src_code = inspect.getsource(callback).strip()

175 else:

176 cls_name = self._type if isinstance(self._type, str) else self._type.__name__ # type: ignore

177 src_code = f"to {cls_name} (missing condition)"

178

179 join = f":{self.join}" if self.join else ""

180 return f"<Relationship{join} {src_code}>"

181

182 def get_table(self, db: "TypeDAL") -> typing.Type["TypedTable"]:

183 """

184 Get the table this relationship is bound to.

185 """

186 table = self.table # can be a string because db wasn't available yet

187 if isinstance(table, str):

188 if mapped := db._class_map.get(table):

189 # yay

190 return mapped

191

192 # boo, fall back to untyped table but pretend it is typed:

193 return typing.cast(typing.Type["TypedTable"], db[table]) # eh close enough!

194

195 return table

196

197 def get_table_name(self) -> str:

198 """

199 Get the name of the table this relationship is bound to.

200 """

201 if isinstance(self.table, str):

202 return self.table

203

204 if isinstance(self.table, pydal.objects.Table):

205 return str(self.table)

206

207 # else: typed table

208 try:

209 table = self.table._ensure_table_defined() if issubclass(self.table, TypedTable) else self.table

210 except Exception: # pragma: no cover

211 table = self.table

212

213 return str(table)

214

215 def __get__(self, instance: Any, owner: Any) -> typing.Optional[list[Any]] | "Relationship[To_Type]":

216 """

217 Relationship is a descriptor class, which can be returned from a class but not an instance.

218

219 For an instance, using .join() will replace the Relationship with the actual data.

220 If you forgot to join, a warning will be shown and empty data will be returned.

221 """

222 if not instance:

223 # relationship queried on class, that's allowed

224 return self

225

226 warnings.warn(

227 "Trying to get data from a relationship object! Did you forget to join it?", category=RuntimeWarning

228 )

229 if self.multiple:

230 return []

231 else:

232 return None

233

234

235 def relationship(

236 _type: To_Type, condition: Condition = None, join: JOIN_OPTIONS = None, on: OnQuery = None

237) -> Relationship[To_Type]:

238 """

239 Define a relationship to another table, when its id is not stored in the current table.

240

241 Example:

242 class User(TypedTable):

243 name: str

244

245 posts = relationship(list["Post"], condition=lambda self, post: self.id == post.author, join='left')

246

247 class Post(TypedTable):

248 title: str

249 author: User

250

251 User.join("posts").first() # User instance with list[Post] in .posts

252

253 Here, Post stores the User ID, but `relationship(list["Post"])` still allows you to get the user's posts.

254 In this case, the join strategy is set to LEFT so users without posts are also still selected.

255

256 For complex queries with a pivot table, a `on` can be set insteaad of `condition`:

257 class User(TypedTable):

258 ...

259

260 tags = relationship(list["Tag"], on=lambda self, tag: [

261 Tagged.on(Tagged.entity == entity.gid),

262 Tag.on((Tagged.tag == tag.id)),

263 ])

264

265 If you'd try to capture this in a single 'condition', pydal would create a cross join which is much less efficient.

266 """

267 return Relationship(_type, condition, join, on)

268

269

270 def _generate_relationship_condition(

271 _: typing.Type["TypedTable"], key: str, field: typing.Union["TypedField[Any]", "Table", typing.Type["TypedTable"]]

272) -> Condition:

273 origin = typing.get_origin(field)

274 # else: generic

275

276 if origin == list:

277 # field = typing.get_args(field)[0] # actual field

278 # return lambda _self, _other: cls[key].contains(field)

279

280 return lambda _self, _other: _self[key].contains(_other.id)

281 else:

282 # normal reference

283 # return lambda _self, _other: cls[key] == field.id

284 return lambda _self, _other: _self[key] == _other.id

285

286

287 def to_relationship(

288 cls: typing.Type["TypedTable"] | type[Any],

289 key: str,

290 field: typing.Union["TypedField[Any]", "Table", typing.Type["TypedTable"]],

291) -> typing.Optional[Relationship[Any]]:

292 """

293 Used to automatically create relationship instance for reference fields.

294

295 Example:

296 class MyTable(TypedTable):

297 reference: OtherTable

298

299 `reference` contains the id of an Other Table row.

300 MyTable.relationships should have 'reference' as a relationship, so `MyTable.join('reference')` should work.

301

302 This function will automatically perform this logic (called in db.define):

303 to_relationship(MyTable, 'reference', OtherTable) -> Relationship[OtherTable]

304

305 Also works for list:reference (list[OtherTable]) and TypedField[OtherTable].

306 """

307 if looks_like(field, TypedField):

308 if args := typing.get_args(field):

309 field = args[0]

310 else:

311 # weird

312 return None

313

314 field, optional = extract_type_optional(field)

315

316 try:

317 condition = _generate_relationship_condition(cls, key, field)

318 except Exception as e: # pragma: no cover

319 warnings.warn("Could not generate Relationship condition", source=e)

320 condition = None

321

322 if not condition: # pragma: no cover

323 # something went wrong, not a valid relationship

324 warnings.warn(f"Invalid relationship for {cls.__name__}.{key}: {field}")

325 return None

326

327 join = "left" if optional or typing.get_origin(field) == list else "inner"

328

329 return Relationship(typing.cast(type[TypedTable], field), condition, typing.cast(JOIN_OPTIONS, join))

330

331

332 class TypeDAL(pydal.DAL): # type: ignore

333 """

334 Drop-in replacement for pyDAL with layer to convert class-based table definitions to classical pydal define_tables.

335 """

336

337 def __init__(

338 self,

339 uri: str | Path = "sqlite://dummy.db",

340 pool_size: int = 0,

341 folder: Optional[str | Path] = None,

342 db_codec: str = "UTF-8",

343 check_reserved: Optional[list[str]] = None,

344 migrate: bool = True,

345 fake_migrate: bool = False,

346 migrate_enabled: bool = True,

347 fake_migrate_all: bool = False,

348 decode_credentials: bool = False,

349 driver_args: Optional[dict[str, Any]] = None,

350 adapter_args: Optional[dict[str, Any]] = None,

351 attempts: int = 5,

352 auto_import: bool = False,

353 bigint_id: bool = False,

354 debug: bool = False,

355 lazy_tables: bool = False,

356 db_uid: Optional[str] = None,

357 after_connection: typing.Callable[..., Any] = None,

358 tables: Optional[list[str]] = None,

359 ignore_field_case: bool = True,

360 entity_quoting: bool = True,

361 table_hash: Optional[str] = None,

362 enable_typedal_caching: bool = True,

363 ) -> None:

364 """

365 Adds some internal tables after calling pydal's default init.

366

367 Set enable_typedal_caching to False to disable this behavior.

368 """

369 if folder:

370 Path(folder).mkdir(exist_ok=True)

371

372 super().__init__(

373 str(uri),

374 pool_size,

375 str(folder),

376 db_codec,

377 check_reserved,

378 migrate,

379 fake_migrate,

380 migrate_enabled,

381 fake_migrate_all,

382 decode_credentials,

383 driver_args,

384 adapter_args,

385 attempts,

386 auto_import,

387 bigint_id,

388 debug,

389 lazy_tables,

390 db_uid,

391 after_connection,

392 tables,

393 ignore_field_case,

394 entity_quoting,

395 table_hash,

396 )

397

398 if enable_typedal_caching:

399 self.try_define(_TypedalCache)

400 self.try_define(_TypedalCacheDependency)

401

402 def try_define(self, model: typing.Type[T]) -> typing.Type[T]:

403 """

404 Try to define a model with migrate or fall back to fake migrate.

405 """

406 try:

407 return self.define(model, migrate=True)

408 except Exception as e:

409 warnings.warn(f"{model} could not be migrated, try faking", source=e, category=RuntimeWarning)

410 return self.define(model, migrate=False, fake_migrate=True, redefine=True)

411

412 default_kwargs: typing.ClassVar[typing.Dict[str, Any]] = {

413 # fields are 'required' (notnull) by default:

414 "notnull": True,

415 }

416

417 # maps table name to typedal class, for resolving future references

418 _class_map: typing.ClassVar[dict[str, typing.Type["TypedTable"]]] = {}

419

420 def _define(self, cls: typing.Type[T], **kwargs: Any) -> typing.Type[T]:

421 # todo: new relationship item added should also invalidate (previously unrelated) cache result

422

423 # todo: option to enable/disable cache dependency behavior:

424 # - don't set _before_update and _before_delete

425 # - don't add TypedalCacheDependency entry

426 # - don't invalidate other item on new row of this type

427

428 # when __future__.annotations is implemented, cls.__annotations__ will not work anymore as below.

429 # proper way to handle this would be (but gives error right now due to Table implementing magic methods):

430 # typing.get_type_hints(cls, globalns=None, localns=None)

431

432 # dirty way (with evil eval):

433 # [eval(v) for k, v in cls.__annotations__.items()]

434 # this however also stops working when variables outside this scope or even references to other

435 # objects are used. So for now, this package will NOT work when from __future__ import annotations is used,

436 # and might break in the future, when this annotations behavior is enabled by default.

437

438 # non-annotated variables have to be passed to define_table as kwargs

439 full_dict = all_dict(cls) # includes properties from parents (e.g. useful for mixins)

440

441 tablename = self.to_snake(cls.__name__)

442 # grab annotations of cls and it's parents:

443 annotations = all_annotations(cls)

444 # extend with `prop = TypedField()` 'annotations':

445 annotations |= {k: typing.cast(type, v) for k, v in full_dict.items() if is_typed_field(v)}

446 # remove internal stuff:

447 annotations = {k: v for k, v in annotations.items() if not k.startswith("_")}

448

449 typedfields: dict[str, TypedField[Any]] = {

450 k: instanciate(v, True) for k, v in annotations.items() if is_typed_field(v)

451 }

452

453 relationships: dict[str, type[Relationship[Any]]] = filter_out(annotations, Relationship)

454

455 fields = {fname: self._to_field(fname, ftype) for fname, ftype in annotations.items()}

456

457 # ! dont' use full_dict here:

458 other_kwargs = kwargs | {

459 k: v for k, v in cls.__dict__.items() if k not in annotations and not k.startswith("_")

460 }

461

462 for key in typedfields.keys() - full_dict.keys():

463 # typed fields that don't haven't been added to the object yet

464 setattr(cls, key, typedfields[key])

465

466 # start with base classes and overwrite with current class:

467 relationships = filter_out(full_dict, Relationship) | relationships | filter_out(other_kwargs, Relationship)

468

469 # DEPRECATED: Relationship as annotation is currently not supported!

470 # ensure they are all instances and

471 # not mix of instances (`= relationship()`) and classes (`: Relationship[...]`):

472 # relationships = {

473 # k: v if isinstance(v, Relationship) else to_relationship(cls, k, v) for k, v in relationships.items()

474 # }

475

476 # keys of implicit references (also relationships):

477 reference_field_keys = [k for k, v in fields.items() if v.type.split(" ")[0] in ("list:reference", "reference")]

478

479 # add implicit relationships:

480 # User; list[User]; TypedField[User]; TypedField[list[User]]

481 relationships |= {

482 k: new_relationship

483 for k in reference_field_keys

484 if k not in relationships and (new_relationship := to_relationship(cls, k, annotations[k]))

485 }

486

487 cache_dependency = other_kwargs.pop("cache_dependency", True)

488

489 table: Table = self.define_table(tablename, *fields.values(), **other_kwargs)

490

491 for name, typed_field in typedfields.items():

492 field = fields[name]

493 typed_field.bind(field, table)

494

495 if issubclass(cls, TypedTable):

496 cls.__set_internals__(

497 db=self,

498 table=table,

499 # by now, all relationships should be instances!

500 relationships=typing.cast(dict[str, Relationship[Any]], relationships),

501 )

502 self._class_map[str(table)] = cls

503 else:

504 warnings.warn("db.define used without inheriting TypedTable. This could lead to strange problems!")

505

506 if not tablename.startswith("typedal_") and cache_dependency:

507 table._before_update.append(lambda s, _: _remove_cache(s, tablename))

508 table._before_delete.append(lambda s: _remove_cache(s, tablename))

509

510 return cls

511

512 @typing.overload

513 def define(self, maybe_cls: None = None, **kwargs: Any) -> typing.Callable[[typing.Type[T]], typing.Type[T]]:

514 """

515 Typing Overload for define without a class.

516

517 @db.define()

518 class MyTable(TypedTable): ...

519 """

520

521 @typing.overload

522 def define(self, maybe_cls: typing.Type[T], **kwargs: Any) -> typing.Type[T]:

523 """

524 Typing Overload for define with a class.

525

526 @db.define

527 class MyTable(TypedTable): ...

528 """

529

530 def define(

531 self, maybe_cls: typing.Type[T] | None = None, **kwargs: Any

532 ) -> typing.Type[T] | typing.Callable[[typing.Type[T]], typing.Type[T]]:

533 """

534 Can be used as a decorator on a class that inherits `TypedTable`, \

535 or as a regular method if you need to define your classes before you have access to a 'db' instance.

536

537 You can also pass extra arguments to db.define_table.

538 See http://www.web2py.com/books/default/chapter/29/06/the-database-abstraction-layer#Table-constructor

539

540 Example:

541 @db.define

542 class Person(TypedTable):

543 ...

544

545 class Article(TypedTable):

546 ...

547

548 # at a later time:

549 db.define(Article)

550

551 Returns:

552 the result of pydal.define_table

553 """

554

555 def wrapper(cls: typing.Type[T]) -> typing.Type[T]:

556 return self._define(cls, **kwargs)

557

558 if maybe_cls:

559 return wrapper(maybe_cls)

560

561 return wrapper

562

563 # def drop(self, table_name: str) -> None:

564 # """

565 # Remove a table by name (both on the database level and the typedal level).

566 # """

567 # # drop calls TypedTable.drop() and removes it from the `_class_map`

568 # if cls := self._class_map.pop(table_name, None):

569 # cls.drop()

570

571 # def drop_all(self, max_retries: int = None) -> None:

572 # """

573 # Remove all tables and keep doing so until everything is gone!

574 # """

575 # retries = 0

576 # if max_retries is None:

577 # max_retries = len(self.tables)

578 #

579 # while self.tables:

580 # retries += 1

581 # for table in self.tables:

582 # self.drop(table)

583 #

584 # if retries > max_retries:

585 # raise RuntimeError("Could not delete all tables")

586

587 def __call__(self, *_args: T_Query, **kwargs: Any) -> "TypedSet":

588 """

589 A db instance can be called directly to perform a query.

590

591 Usually, only a query is passed.

592

593 Example:

594 db(query).select()

595

596 """

597 args = list(_args)

598 if args:

599 cls = args[0]

600 if isinstance(cls, bool):

601 raise ValueError("Don't actually pass a bool to db()! Use a query instead.")

602

603 if isinstance(cls, type) and issubclass(type(cls), type) and issubclass(cls, TypedTable):

604 # table defined without @db.define decorator!

605 _cls: typing.Type[TypedTable] = cls

606 args[0] = _cls.id != None

607

608 _set = super().__call__(*args, **kwargs)

609 return typing.cast(TypedSet, _set)

610

611 @classmethod

612 def _build_field(cls, name: str, _type: str, **kw: Any) -> Field:

613 return Field(name, _type, **{**cls.default_kwargs, **kw})

614

615 @classmethod

616 def _annotation_to_pydal_fieldtype(

617 cls, _ftype: T_annotation, mut_kw: typing.MutableMapping[str, Any]

618 ) -> Optional[str]:

619 # ftype can be a union or type. typing.cast is sometimes used to tell mypy when it's not a union.

620 ftype = typing.cast(type, _ftype) # cast from typing.Type to type to make mypy happy)

621

622 if isinstance(ftype, str):

623 # extract type from string

624 ftype = typing.get_args(typing.Type[ftype])[0]._evaluate(

625 localns=locals(), globalns=globals(), recursive_guard=frozenset()

626 )

627

628 if mapping := BASIC_MAPPINGS.get(ftype):

629 # basi types

630 return mapping

631 elif isinstance(ftype, _Table):

632 # db.table

633 return f"reference {ftype._tablename}"

634 elif issubclass(type(ftype), type) and issubclass(ftype, TypedTable):

635 # SomeTable

636 snakename = cls.to_snake(ftype.__name__)

637 return f"reference {snakename}"

638 elif isinstance(ftype, TypedField):

639 # FieldType(type, ...)

640 return ftype._to_field(mut_kw)

641 elif origin_is_subclass(ftype, TypedField):

642 # TypedField[int]

643 return cls._annotation_to_pydal_fieldtype(typing.get_args(ftype)[0], mut_kw)

644 elif isinstance(ftype, types.GenericAlias) and typing.get_origin(ftype) in (list, TypedField):

645 # list[str] -> str -> string -> list:string

646 _child_type = typing.get_args(ftype)[0]

647 _child_type = cls._annotation_to_pydal_fieldtype(_child_type, mut_kw)

648 return f"list:{_child_type}"

649 elif is_union(ftype):

650 # str | int -> UnionType

651 # typing.Union[str | int] -> typing._UnionGenericAlias

652

653 # Optional[type] == type | None

654

655 match typing.get_args(ftype):

656 case (_child_type, _Types.NONETYPE) | (_Types.NONETYPE, _child_type):

657 # good union of Nullable

658

659 # if a field is optional, it is nullable:

660 mut_kw["notnull"] = False

661 return cls._annotation_to_pydal_fieldtype(_child_type, mut_kw)

662 case _:

663 # two types is not supported by the db!

664 return None

665 else:

666 return None

667

668 @classmethod

669 def _to_field(cls, fname: str, ftype: type, **kw: Any) -> Field:

670 """

671 Convert a annotation into a pydal Field.

672

673 Args:

674 fname: name of the property

675 ftype: annotation of the property

676 kw: when using TypedField or a function returning it (e.g. StringField),

677 keyword args can be used to pass any other settings you would normally to a pydal Field

678

679 -> pydal.Field(fname, ftype, **kw)

680

681 Example:

682 class MyTable:

683 fname: ftype

684 id: int

685 name: str

686 reference: Table

687 other: TypedField(str, default="John Doe") # default will be in kwargs

688 """

689 fname = cls.to_snake(fname)

690

691 if converted_type := cls._annotation_to_pydal_fieldtype(ftype, kw):

692 return cls._build_field(fname, converted_type, **kw)

693 else:

694 raise NotImplementedError(f"Unsupported type {ftype}/{type(ftype)}")

695

696 @staticmethod

697 def to_snake(camel: str) -> str:

698 """

699 Moved to helpers, kept as a static method for legacy reasons.

700 """

701 return to_snake(camel)

702

703

704 class TableProtocol(typing.Protocol): # pragma: no cover

705 """

706 Make mypy happy.

707 """

708

709 id: int # noqa: A003

710

711 def __getitem__(self, item: str) -> Field:

712 """

713 Tell mypy a Table supports dictionary notation for columns.

714 """

715

716

717 class Table(_Table, TableProtocol): # type: ignore

718 """

719 Make mypy happy.

720 """

721

722

723 class TableMeta(type):

724 """

725 This metaclass contains functionality on table classes, that doesn't exist on its instances.

726

727 Example:

728 class MyTable(TypedTable):

729 some_field: TypedField[int]

730

731 MyTable.update_or_insert(...) # should work

732

733 MyTable.some_field # -> Field, can be used to query etc.

734

735 row = MyTable.first() # returns instance of MyTable

736

737 # row.update_or_insert(...) # shouldn't work!

738

739 row.some_field # -> int, with actual data

740

741 """

742

743 # set up by db.define:

744 # _db: TypeDAL | None = None

745 # _table: Table | None = None

746 _db: TypeDAL | None = None

747 _table: Table | None = None

748 _relationships: dict[str, Relationship[Any]] | None = None

749

750 #########################

751 # TypeDAL custom logic: #

752 #########################

753

754 def __set_internals__(self, db: pydal.DAL, table: Table, relationships: dict[str, Relationship[Any]]) -> None:

755 """

756 Store the related database and pydal table for later usage.

757 """

758 self._db = db

759 self._table = table

760 self._relationships = relationships

761

762 def __getattr__(self, col: str) -> Optional[Field]:

763 """

764 Magic method used by TypedTableMeta to get a database field with dot notation on a class.

765

766 Example:

767 SomeTypedTable.col -> db.table.col (via TypedTableMeta.__getattr__)

768

769 """

770 if self._table:

771 return getattr(self._table, col, None)

772

773 return None

774

775 def _ensure_table_defined(self) -> Table:

776 if not self._table:

777 raise EnvironmentError("@define or db.define is not called on this class yet!")

778 return self._table

779

780 def __iter__(self) -> typing.Generator[Field, None, None]:

781 """

782 Loop through the columns of this model.

783 """

784 table = self._ensure_table_defined()

785 yield from iter(table)

786

787 def __getitem__(self, item: str) -> Field:

788 """

789 Allow dict notation to get a column of this table (-> Field instance).

790 """

791 table = self._ensure_table_defined()

792 return table[item]

793

794 def __str__(self) -> str:

795 """

796 Normally, just returns the underlying table name, but with a fallback if the model is unbound.

797 """

798 if self._table:

799 return str(self._table)

800 else:

801 return f"<unbound table {self.__name__}>"

802

803 def from_row(self: typing.Type[T_MetaInstance], row: pydal.objects.Row) -> T_MetaInstance:

804 """

805 Create a model instance from a pydal row.

806 """

807 return self(row)

808

809 def all(self: typing.Type[T_MetaInstance]) -> "TypedRows[T_MetaInstance]": # noqa: A003

810 """

811 Return all rows for this model.

812 """

813 return self.collect()

814

815 def __json__(self: typing.Type[T_MetaInstance], instance: T_MetaInstance | None = None) -> dict[str, Any]:

816 """

817 Convert to a json-dumpable dict.

818

819 as_dict is not fully json-dumpable, so use as_json and json.loads to ensure it is dumpable (and loadable).

820 todo: can this be optimized?

821

822 See Also:

823 https://github.com/jeff-hykin/json_fix

824 """

825 string = instance.as_json() if instance else self.as_json()

826

827 return typing.cast(dict[str, Any], json.loads(string))

828

829 def get_relationships(self) -> dict[str, Relationship[Any]]:

830 """

831 Return the registered relationships of the current model.

832 """

833 return self._relationships or {}

834

835 ##########################

836 # TypeDAL Modified Logic #

837 ##########################

838

839 def insert(self: typing.Type[T_MetaInstance], **fields: Any) -> T_MetaInstance:

840 """

841 This is only called when db.define is not used as a decorator.

842

843 cls.__table functions as 'self'

844

845 Args:

846 **fields: anything you want to insert in the database

847

848 Returns: the ID of the new row.

849

850 """

851 table = self._ensure_table_defined()

852

853 result = table.insert(**fields)

854 # it already is an int but mypy doesn't understand that

855 return self(result)

856

857 def _insert(self, **fields: Any) -> str:

858 table = self._ensure_table_defined()

859

860 return str(table._insert(**fields))

861

862 def bulk_insert(self: typing.Type[T_MetaInstance], items: list[dict[str, Any]]) -> "TypedRows[T_MetaInstance]":

863 """

864 Insert multiple rows, returns a TypedRows set of new instances.

865 """

866 table = self._ensure_table_defined()

867 result = table.bulk_insert(items)

868 return self.where(lambda row: row.id.belongs(result)).collect()

869

870 def update_or_insert(

871 self: typing.Type[T_MetaInstance], query: T_Query | dict[str, Any] = DEFAULT, **values: Any

872 ) -> T_MetaInstance:

873 """

874 Update a row if query matches, else insert a new one.

875

876 Returns the created or updated instance.

877 """

878 table = self._ensure_table_defined()

879

880 if query is DEFAULT:

881 record = table(**values)

882 elif isinstance(query, dict):

883 record = table(**query)

884 else:

885 record = table(query)

886

887 if not record:

888 return self.insert(**values)

889

890 record.update_record(**values)

891 return self(record)

892

893 def validate_and_insert(

894 self: typing.Type[T_MetaInstance], **fields: Any

895 ) -> tuple[Optional[T_MetaInstance], Optional[dict[str, str]]]:

896 """

897 Validate input data and then insert a row.

898

899 Returns a tuple of (the created instance, a dict of errors).

900 """

901 table = self._ensure_table_defined()

902 result = table.validate_and_insert(**fields)

903 if row_id := result.get("id"):

904 return self(row_id), None

905 else:

906 return None, result.get("errors")

907

908 def validate_and_update(

909 self: typing.Type[T_MetaInstance], query: Query, **fields: Any

910 ) -> tuple[Optional[T_MetaInstance], Optional[dict[str, str]]]:

911 """

912 Validate input data and then update max 1 row.

913

914 Returns a tuple of (the updated instance, a dict of errors).

915 """

916 table = self._ensure_table_defined()

917

918 try:

919 result = table.validate_and_update(query, **fields)

920 except Exception as e:

921 result = {"errors": {"exception": str(e)}}

922

923 if errors := result.get("errors"):

924 return None, errors

925 elif row_id := result.get("id"):

926 return self(row_id), None

927 else: # pragma: no cover

928 # update on query without result (shouldnt happen)

929 return None, None

930

931 def validate_and_update_or_insert(

932 self: typing.Type[T_MetaInstance], query: Query, **fields: Any

933 ) -> tuple[Optional[T_MetaInstance], Optional[dict[str, str]]]:

934 """

935 Validate input data and then update_and_insert (on max 1 row).

936

937 Returns a tuple of (the updated/created instance, a dict of errors).

938 """

939 table = self._ensure_table_defined()

940 result = table.validate_and_update_or_insert(query, **fields)

941

942 if errors := result.get("errors"):

943 return None, errors

944 elif row_id := result.get("id"):

945 return self(row_id), None

946 else: # pragma: no cover

947 # update on query without result (shouldnt happen)

948 return None, None

949

950 def select(self: typing.Type[T_MetaInstance], *a: Any, **kw: Any) -> "QueryBuilder[T_MetaInstance]":

951 """

952 See QueryBuilder.select!

953 """

954 return QueryBuilder(self).select(*a, **kw)

955

956 def paginate(self: typing.Type[T_MetaInstance], limit: int, page: int = 1) -> "PaginatedRows[T_MetaInstance]":

957 """

958 See QueryBuilder.paginate!

959 """

960 return QueryBuilder(self).paginate(limit=limit, page=page)

961

962 def chunk(

963 self: typing.Type[T_MetaInstance], chunk_size: int

964 ) -> typing.Generator["TypedRows[T_MetaInstance]", Any, None]:

965 """

966 See QueryBuilder.chunk!

967 """

968 return QueryBuilder(self).chunk(chunk_size)

969

970 def where(self: typing.Type[T_MetaInstance], *a: Any, **kw: Any) -> "QueryBuilder[T_MetaInstance]":

971 """

972 See QueryBuilder.where!

973 """

974 return QueryBuilder(self).where(*a, **kw)

975

976 def cache(self: typing.Type[T_MetaInstance], *deps: Any, **kwargs: Any) -> "QueryBuilder[T_MetaInstance]":

977 """

978 See QueryBuilder.cache!

979 """

980 return QueryBuilder(self).cache(*deps, **kwargs)

981

982 def count(self: typing.Type[T_MetaInstance]) -> int:

983 """

984 See QueryBuilder.count!

985 """

986 return QueryBuilder(self).count()

987

988 def first(self: typing.Type[T_MetaInstance]) -> T_MetaInstance | None:

989 """

990 See QueryBuilder.first!

991 """

992 return QueryBuilder(self).first()

993

994 def join(

995 self: typing.Type[T_MetaInstance],

996 *fields: str | typing.Type["TypedTable"],

997 method: JOIN_OPTIONS = None,

998 on: OnQuery | list[Expression] | Expression = None,

999 condition: Condition = None,

1000 ) -> "QueryBuilder[T_MetaInstance]":

1001 """

1002 See QueryBuilder.join!

1003 """

1004 return QueryBuilder(self).join(*fields, on=on, condition=condition, method=method)

1005

1006 def collect(self: typing.Type[T_MetaInstance], verbose: bool = False) -> "TypedRows[T_MetaInstance]":

1007 """

1008 See QueryBuilder.collect!

1009 """

1010 return QueryBuilder(self).collect(verbose=verbose)

1011

1012 @property

1013 def ALL(cls) -> pydal.objects.SQLALL:

1014 """

1015 Select all fields for this table.

1016 """

1017 table = cls._ensure_table_defined()

1018

1019 return table.ALL

1020

1021 ##########################

1022 # TypeDAL Shadowed Logic #

1023 ##########################

1024 fields: list[str]

1025

1026 # other table methods:

1027

1028 def drop(self, mode: str = "") -> None:

1029 """

1030 Remove the underlying table.

1031 """

1032 table = self._ensure_table_defined()

1033 table.drop(mode)

1034

1035 def create_index(self, name: str, *fields: Field | str, **kwargs: Any) -> bool:

1036 """

1037 Add an index on some columns of this table.

1038 """

1039 table = self._ensure_table_defined()

1040 result = table.create_index(name, *fields, **kwargs)

1041 return typing.cast(bool, result)

1042

1043 def drop_index(self, name: str, if_exists: bool = False) -> bool:

1044 """

1045 Remove an index from this table.

1046 """

1047 table = self._ensure_table_defined()

1048 result = table.drop_index(name, if_exists)

1049 return typing.cast(bool, result)

1050

1051 def import_from_csv_file(

1052 self,

1053 csvfile: typing.TextIO,

1054 id_map: dict[str, str] = None,

1055 null: Any = "<NULL>",

1056 unique: str = "uuid",

1057 id_offset: dict[str, int] = None, # id_offset used only when id_map is None

1058 transform: typing.Callable[[dict[Any, Any]], dict[Any, Any]] = None,

1059 validate: bool = False,

1060 encoding: str = "utf-8",

1061 delimiter: str = ",",

1062 quotechar: str = '"',

1063 quoting: int = csv.QUOTE_MINIMAL,

1064 restore: bool = False,

1065 **kwargs: Any,

1066 ) -> None:

1067 """

1068 Load a csv file into the database.

1069 """

1070 table = self._ensure_table_defined()

1071 table.import_from_csv_file(

1072 csvfile,

1073 id_map=id_map,

1074 null=null,

1075 unique=unique,

1076 id_offset=id_offset,

1077 transform=transform,

1078 validate=validate,

1079 encoding=encoding,

1080 delimiter=delimiter,

1081 quotechar=quotechar,

1082 quoting=quoting,

1083 restore=restore,

1084 **kwargs,

1085 )

1086

1087 def on(self, query: Query | bool) -> Expression:

1088 """

1089 Shadow Table.on.

1090

1091 Used for joins.

1092

1093 See Also:

1094 http://web2py.com/books/default/chapter/29/06/the-database-abstraction-layer?search=export_to_csv_file#One-to-many-relation

1095 """

1096 table = self._ensure_table_defined()

1097 return typing.cast(Expression, table.on(query))

1098

1099 def with_alias(self, alias: str) -> _Table:

1100 """

1101 Shadow Table.with_alias.

1102

1103 Useful for joins when joining the same table multiple times.

1104

1105 See Also:

1106 http://web2py.com/books/default/chapter/29/06/the-database-abstraction-layer?search=export_to_csv_file#One-to-many-relation

1107 """

1108 table = self._ensure_table_defined()

1109 return table.with_alias(alias)

1110

1111 # @typing.dataclass_transform()

1112

1113

1114 class TypedField(typing.Generic[T_Value]): # pragma: no cover

1115 """

1116 Typed version of pydal.Field, which will be converted to a normal Field in the background.

1117 """

1118

1119 # will be set by .bind on db.define

1120 name = ""

1121 _db: Optional[pydal.DAL] = None

1122 _rname: Optional[str] = None

1123 _table: Optional[Table] = None

1124 _field: Optional[Field] = None

1125

1126 _type: T_annotation

1127 kwargs: Any

1128

1129 def __init__(self, _type: typing.Type[T_Value] | types.UnionType = str, /, **settings: Any) -> None: # type: ignore

1130 """

1131 A TypedFieldType should not be inited manually, but TypedField (from `fields.py`) should be used!

1132 """

1133 self._type = _type

1134 self.kwargs = settings

1135 super().__init__()

1136

1137 @typing.overload

1138 def __get__(self, instance: T_MetaInstance, owner: typing.Type[T_MetaInstance]) -> T_Value: # pragma: no cover

1139 """

1140 row.field -> (actual data).

1141 """

1142

1143 @typing.overload

1144 def __get__(self, instance: None, owner: "typing.Type[TypedTable]") -> "TypedField[T_Value]": # pragma: no cover

1145 """

1146 Table.field -> Field.

1147 """

1148

1149 def __get__(

1150 self, instance: T_MetaInstance | None, owner: typing.Type[T_MetaInstance]

1151 ) -> typing.Union[T_Value, "TypedField[T_Value]"]:

1152 """

1153 Since this class is a Descriptor field, \

1154 it returns something else depending on if it's called on a class or instance.

1155

1156 (this is mostly for mypy/typing)

1157 """

1158 if instance:

1159 # this is only reached in a very specific case:

1160 # an instance of the object was created with a specific set of fields selected (excluding the current one)

1161 # in that case, no value was stored in the owner -> return None (since the field was not selected)

1162 return typing.cast(T_Value, None) # cast as T_Value so mypy understands it for selected fields

1163 else:

1164 # getting as class -> return actual field so pydal understands it when using in query etc.

1165 return typing.cast(TypedField[T_Value], self._field) # pretend it's still typed for IDE support

1166

1167 def __str__(self) -> str:

1168 """

1169 String representation of a Typed Field.

1170

1171 If `type` is set explicitly (e.g. TypedField(str, type="text")), that type is used: `TypedField.text`,

1172 otherwise the type annotation is used (e.g. TypedField(str) -> TypedField.str)

1173 """

1174 return str(self._field) if self._field else ""

1175

1176 def __repr__(self) -> str:

1177 """

1178 More detailed string representation of a Typed Field.

1179

1180 Uses __str__ and adds the provided extra options (kwargs) in the representation.

1181 """

1182 s = self.__str__()

1183

1184 if "type" in self.kwargs:

1185 # manual type in kwargs supplied

1186 t = self.kwargs["type"]

1187 elif issubclass(type, type(self._type)):

1188 # normal type, str.__name__ = 'str'

1189 t = getattr(self._type, "__name__", str(self._type))

1190 elif t_args := typing.get_args(self._type):

1191 # list[str] -> 'str'

1192 t = t_args[0].__name__

1193 else: # pragma: no cover

1194 # fallback - something else, may not even happen, I'm not sure

1195 t = self._type

1196

1197 s = f"TypedField[{t}].{s}" if s else f"TypedField[{t}]"

1198

1199 kw = self.kwargs.copy()

1200 kw.pop("type", None)

1201 return f"<{s} with options {kw}>"

1202

1203 def _to_field(self, extra_kwargs: typing.MutableMapping[str, Any]) -> Optional[str]:

1204 """

1205 Convert a Typed Field instance to a pydal.Field.

1206 """

1207 other_kwargs = self.kwargs.copy()

1208 extra_kwargs.update(other_kwargs)

1209 return extra_kwargs.pop("type", False) or TypeDAL._annotation_to_pydal_fieldtype(self._type, extra_kwargs)

1210

1211 def bind(self, field: pydal.objects.Field, table: pydal.objects.Table) -> None:

1212 """

1213 Bind the right db/table/field info to this class, so queries can be made using `Class.field == ...`.

1214 """

1215 self._table = table

1216 self._field = field

1217

1218 def __getattr__(self, key: str) -> Any:

1219 """

1220 If the regular getattribute does not work, try to get info from the related Field.

1221 """

1222 with contextlib.suppress(AttributeError):

1223 return super().__getattribute__(key)

1224

1225 # try on actual field:

1226 return getattr(self._field, key)

1227

1228 def __eq__(self, other: Any) -> Query:

1229 """

1230 Performing == on a Field will result in a Query.

1231 """

1232 return typing.cast(Query, self._field == other)

1233

1234 def __ne__(self, other: Any) -> Query:

1235 """

1236 Performing != on a Field will result in a Query.

1237 """

1238 return typing.cast(Query, self._field != other)

1239

1240 def __gt__(self, other: Any) -> Query:

1241 """

1242 Performing > on a Field will result in a Query.

1243 """

1244 return typing.cast(Query, self._field > other)

1245

1246 def __lt__(self, other: Any) -> Query:

1247 """

1248 Performing < on a Field will result in a Query.

1249 """

1250 return typing.cast(Query, self._field < other)

1251

1252 def __ge__(self, other: Any) -> Query:

1253 """

1254 Performing >= on a Field will result in a Query.

1255 """

1256 return typing.cast(Query, self._field >= other)

1257

1258 def __le__(self, other: Any) -> Query:

1259 """

1260 Performing <= on a Field will result in a Query.

1261 """

1262 return typing.cast(Query, self._field <= other)

1263

1264 def __hash__(self) -> int:

1265 """

1266 Shadow Field.__hash__.

1267 """

1268 return hash(self._field)

1269

1270 def __invert__(self) -> Expression:

1271 """

1272 Performing ~ on a Field will result in an Expression.

1273 """

1274 if not self._field: # pragma: no cover

1275 raise ValueError("Unbound Field can not be inverted!")

1276

1277 return typing.cast(Expression, ~self._field)

1278

1279

1280 class TypedTable(metaclass=TableMeta):

1281 """

1282 Enhanded modeling system on top of pydal's Table that adds typing and additional functionality.

1283 """

1284

1285 # set up by 'new':

1286 _row: Row | None = None

1287

1288 _with: list[str]

1289

1290 id: "TypedField[int]" # noqa: A003

1291

1292 _before_insert: list[BeforeInsertCallable]

1293 _after_insert: list[AfterInsertCallable]

1294 _before_update: list[BeforeUpdateCallable]

1295 _after_update: list[AfterUpdateCallable]

1296 _before_delete: list[BeforeDeleteCallable]

1297 _after_delete: list[AfterDeleteCallable]

1298

1299 def _setup_instance_methods(self) -> None:

1300 self.as_dict = self._as_dict # type: ignore

1301 self.__json__ = self.as_json = self._as_json # type: ignore

1302 # self.as_yaml = self._as_yaml # type: ignore

1303 self.as_xml = self._as_xml # type: ignore

1304

1305 self.update = self._update # type: ignore

1306

1307 self.delete_record = self._delete_record # type: ignore

1308 self.update_record = self._update_record # type: ignore

1309

1310 def __new__(

1311 cls, row_or_id: typing.Union[Row, Query, pydal.objects.Set, int, str, None, "TypedTable"] = None, **filters: Any

1312 ) -> "TypedTable":

1313 """

1314 Create a Typed Rows model instance from an existing row, ID or query.

1315

1316 Examples:

1317 MyTable(1)

1318 MyTable(id=1)

1319 MyTable(MyTable.id == 1)

1320 """

1321 table = cls._ensure_table_defined()

1322 inst = super().__new__(cls)

1323

1324 if isinstance(row_or_id, TypedTable):

1325 # existing typed table instance!

1326 return row_or_id

1327 elif isinstance(row_or_id, pydal.objects.Row):

1328 row = row_or_id

1329 elif row_or_id is not None:

1330 row = table(row_or_id, **filters)

1331 elif filters:

1332 row = table(**filters)

1333 else:

1334 # dummy object

1335 return inst

1336

1337 if not row:

1338 return None # type: ignore

1339

1340 inst._row = row

1341 inst.__dict__.update(row)

1342 inst._setup_instance_methods()

1343 return inst

1344

1345 def __iter__(self) -> typing.Generator[Any, None, None]:

1346 """

1347 Allows looping through the columns.

1348 """

1349 row = self._ensure_matching_row()

1350 yield from iter(row)

1351

1352 def __getitem__(self, item: str) -> Any:

1353 """

1354 Allows dictionary notation to get columns.

1355 """

1356 if item in self.__dict__:

1357 return self.__dict__.get(item)

1358

1359 # fallback to lookup in row

1360 if self._row:

1361 return self._row[item]

1362

1363 # nothing found!

1364 raise KeyError(item)

1365

1366 def __getattr__(self, item: str) -> Any:

1367 """

1368 Allows dot notation to get columns.

1369 """

1370 if value := self.get(item):

1371 return value

1372

1373 raise AttributeError(item)

1374

1375 def get(self, item: str, default: Any = None) -> Any:

1376 """

1377 Try to get a column from this instance, else return default.

1378 """

1379 try:

1380 return self.__getitem__(item)

1381 except KeyError:

1382 return default

1383

1384 def __setitem__(self, key: str, value: Any) -> None:

1385 """

1386 Data can both be updated via dot and dict notation.

1387 """

1388 return setattr(self, key, value)

1389

1390 def __int__(self) -> int:

1391 """

1392 Calling int on a model instance will return its id.

1393 """

1394 return getattr(self, "id", 0)

1395

1396 def __bool__(self) -> bool:

1397 """

1398 If the instance has an underlying row with data, it is truthy.

1399 """

1400 return bool(getattr(self, "_row", False))

1401

1402 def _ensure_matching_row(self) -> Row:

1403 if not getattr(self, "_row", None):

1404 raise EnvironmentError("Trying to access non-existant row. Maybe it was deleted or not yet initialized?")

1405 return self._row

1406

1407 def __repr__(self) -> str:

1408 """

1409 String representation of the model instance.

1410 """

1411 model_name = self.__class__.__name__

1412 model_data = {}

1413

1414 if self._row:

1415 model_data = self._row.as_json()

1416

1417 details = model_name

1418 details += f"({model_data})"

1419

1420 if relationships := getattr(self, "_with", []):

1421 details += f" + {relationships}"

1422

1423 return f"<{details}>"

1424

1425 # serialization

1426 # underscore variants work for class instances (set up by _setup_instance_methods)

1427

1428 @classmethod

1429 def as_dict(cls, flat: bool = False, sanitize: bool = True) -> dict[str, Any]:

1430 """

1431 Dump the object to a plain dict.

1432

1433 Can be used as both a class or instance method:

1434 - dumps the table info if it's a class

1435 - dumps the row info if it's an instance (see _as_dict)

1436 """

1437 table = cls._ensure_table_defined()

1438 result = table.as_dict(flat, sanitize)

1439 return typing.cast(dict[str, Any], result)

1440

1441 @classmethod

1442 def as_json(cls, sanitize: bool = True) -> str:

1443 """

1444 Dump the object to json.

1445

1446 Can be used as both a class or instance method:

1447 - dumps the table info if it's a class

1448 - dumps the row info if it's an instance (see _as_json)

1449 """

1450 table = cls._ensure_table_defined()

1451 return typing.cast(str, table.as_json(sanitize))

1452

1453 @classmethod

1454 def as_xml(cls, sanitize: bool = True) -> str: # pragma: no cover

1455 """

1456 Dump the object to xml.

1457

1458 Can be used as both a class or instance method:

1459 - dumps the table info if it's a class

1460 - dumps the row info if it's an instance (see _as_xml)

1461 """

1462 table = cls._ensure_table_defined()

1463 return typing.cast(str, table.as_xml(sanitize))

1464

1465 @classmethod

1466 def as_yaml(cls, sanitize: bool = True) -> str:

1467 """

1468 Dump the object to yaml.

1469

1470 Can be used as both a class or instance method:

1471 - dumps the table info if it's a class

1472 - dumps the row info if it's an instance (see _as_yaml)

1473 """

1474 table = cls._ensure_table_defined()

1475 return typing.cast(str, table.as_yaml(sanitize))

1476

1477 def _as_dict(

1478 self, datetime_to_str: bool = False, custom_types: typing.Iterable[type] | type | None = None

1479 ) -> dict[str, Any]:

1480 row = self._ensure_matching_row()

1481 result = row.as_dict(datetime_to_str=datetime_to_str, custom_types=custom_types)

1482

1483 if _with := getattr(self, "_with", None):

1484 for relationship in _with:

1485 data = self.get(relationship)

1486 if isinstance(data, list):

1487 data = [_.as_dict() if getattr(_, "as_dict", None) else _ for _ in data]

1488 elif data:

1489 data = data.as_dict()

1490

1491 result[relationship] = data

1492

1493 return typing.cast(dict[str, Any], result)

1494

1495 def _as_json(

1496 self,

1497 mode: str = "object",

1498 default: typing.Callable[[Any], Any] = None,

1499 colnames: list[str] = None,

1500 serialize: bool = True,

1501 **kwargs: Any,

1502 ) -> str:

1503 row = self._ensure_matching_row()

1504 return typing.cast(str, row.as_json(mode, default, colnames, serialize, *kwargs))

1505

1506 def _as_xml(self, sanitize: bool = True) -> str: # pragma: no cover

1507 row = self._ensure_matching_row()

1508 return typing.cast(str, row.as_xml(sanitize))

1509

1510 # def _as_yaml(self, sanitize: bool = True) -> str:

1511 # row = self._ensure_matching_row()

1512 # return typing.cast(str, row.as_yaml(sanitize))

1513

1514 def __setattr__(self, key: str, value: Any) -> None:

1515 """

1516 When setting a property on a Typed Table model instance, also update the underlying row.

1517 """

1518 if self._row and key in self._row.__dict__ and not callable(value):

1519 # enables `row.key = value; row.update_record()`

1520 self._row[key] = value

1521

1522 super().__setattr__(key, value)

1523

1524 @classmethod

1525 def update(cls: typing.Type[T_MetaInstance], query: Query, **fields: Any) -> T_MetaInstance | None:

1526 """

1527 Update one record.

1528

1529 Example:

1530 MyTable.update(MyTable.id == 1, name="NewName") -> MyTable

1531 """

1532 # todo: update multiple?

1533 if record := cls(query):

1534 return record.update_record(**fields)

1535 else:

1536 return None

1537

1538 def _update(self: T_MetaInstance, **fields: Any) -> T_MetaInstance:

1539 row = self._ensure_matching_row()

1540 row.update(**fields)

1541 self.__dict__.update(**fields)

1542 return self

1543

1544 def _update_record(self: T_MetaInstance, **fields: Any) -> T_MetaInstance:

1545 row = self._ensure_matching_row()

1546 new_row = row.update_record(**fields)

1547 self.update(**new_row)

1548 return self

1549

1550 def update_record(self: T_MetaInstance, **fields: Any) -> T_MetaInstance: # pragma: no cover

1551 """

1552 Here as a placeholder for _update_record.

1553

1554 Will be replaced on instance creation!

1555 """

1556 return self._update_record(**fields)

1557

1558 def _delete_record(self) -> int:

1559 """

1560 Actual logic in `pydal.helpers.classes.RecordDeleter`.

1561 """

1562 row = self._ensure_matching_row()

1563 result = row.delete_record()

1564 self.__dict__ = {} # empty self, since row is no more.

1565 self._row = None # just to be sure

1566 self._setup_instance_methods()

1567 # ^ instance methods might've been deleted by emptying dict,

1568 # but we still want .as_dict to show an error, not the table's as_dict.

1569 return typing.cast(int, result)

1570

1571 def delete_record(self) -> int: # pragma: no cover

1572 """

1573 Here as a placeholder for _delete_record.

1574

1575 Will be replaced on instance creation!

1576 """

1577 return self._delete_record()

1578

1579 # __del__ is also called on the end of a scope so don't remove records on every del!!

1580

1581 # pickling:

1582 def __setstate__(self, state: dict[str, Any]) -> None:

1583 """

1584 Used by dill when loading from a bytestring.

1585 """

1586 # as_dict also includes table info, so dump as json to only get the actual row data

1587 # then create a new (more empty) row object:

1588 state["_row"] = Row(json.loads(state["_row"]))

1589 self.__dict__ |= state

1590

1591 def __getstate__(self) -> dict[str, Any]:

1592 """

1593 State to save when pickling.

1594

1595 Prevents db connection from being pickled.

1596 Similar to as_dict but without changing the data of the relationships (dill does that recursively)

1597 """

1598 row = self._ensure_matching_row()

1599 result: dict[str, Any] = row.as_dict()

1600

1601 if _with := getattr(self, "_with", None):

1602 for relationship in _with:

1603 data = self.get(relationship)

1604

1605 result[relationship] = data

1606

1607 result["_row"] = self._row.as_json() if self._row else ""

1608 return result

1609

1610

1611# backwards compat:

1612 TypedRow = TypedTable

1613

1614

1615 class TypedRows(typing.Collection[T_MetaInstance], Rows):

1616 """

1617 Slighly enhaned and typed functionality on top of pydal Rows (the result of a select).

1618 """

1619

1620 records: dict[int, T_MetaInstance]

1621 # _rows: Rows

1622 model: typing.Type[T_MetaInstance]

1623 metadata: Metadata

1624

1625 # pseudo-properties: actually stored in _rows

1626 db: TypeDAL

1627 colnames: list[str]

1628 fields: list[Field]

1629 colnames_fields: list[Field]

1630 response: list[tuple[Any, ...]]

1631

1632 def __init__(

1633 self,

1634 rows: Rows,

1635 model: typing.Type[T_MetaInstance],

1636 records: dict[int, T_MetaInstance] = None,

1637 metadata: Metadata = None,

1638 ) -> None:

1639 """

1640 Should not be called manually!

1641

1642 Normally, the `records` from an existing `Rows` object are used

1643 but these can be overwritten with a `records` dict.

1644 `metadata` can be any (un)structured data

1645 `model` is a Typed Table class

1646 """

1647 records = records or {row.id: model(row) for row in rows}

1648 super().__init__(rows.db, records, rows.colnames, rows.compact, rows.response, rows.fields)

1649 self.model = model

1650 self.metadata = metadata or {}

1651

1652 def __len__(self) -> int:

1653 """

1654 Return the count of rows.

1655 """

1656 return len(self.records)

1657

1658 def __iter__(self) -> typing.Iterator[T_MetaInstance]:

1659 """

1660 Loop through the rows.

1661 """

1662 yield from self.records.values()

1663

1664 def __contains__(self, ind: Any) -> bool:

1665 """

1666 Check if an id exists in this result set.

1667 """

1668 return ind in self.records

1669

1670 def first(self) -> T_MetaInstance | None:

1671 """

1672 Get the row with the lowest id.

1673 """

1674 if not self.records:

1675 return None

1676

1677 return next(iter(self))

1678

1679 def last(self) -> T_MetaInstance | None:

1680 """

1681 Get the row with the highest id.

1682 """

1683 if not self.records:

1684 return None

1685

1686 max_id = max(self.records.keys())

1687 return self[max_id]

1688

1689 def find(

1690 self, f: typing.Callable[[T_MetaInstance], Query], limitby: tuple[int, int] = None

1691 ) -> "TypedRows[T_MetaInstance]":

1692 """

1693 Returns a new Rows object, a subset of the original object, filtered by the function `f`.

1694 """

1695 if not self.records:

1696 return self.__class__(self, self.model, {})

1697

1698 records = {}

1699 if limitby:

1700 _min, _max = limitby

1701 else:

1702 _min, _max = 0, len(self)

1703 count = 0

1704 for i, row in self.records.items():

1705 if f(row):

1706 if _min <= count:

1707 records[i] = row

1708 count += 1

1709 if count == _max:

1710 break

1711

1712 return self.__class__(self, self.model, records)

1713

1714 def exclude(self, f: typing.Callable[[T_MetaInstance], Query]) -> "TypedRows[T_MetaInstance]":

1715 """

1716 Removes elements from the calling Rows object, filtered by the function `f`, \

1717 and returns a new Rows object containing the removed elements.

1718 """

1719 if not self.records:

1720 return self.__class__(self, self.model, {})

1721 removed = {}

1722 to_remove = []

1723 for i in self.records:

1724 row = self[i]

1725 if f(row):

1726 removed[i] = self.records[i]

1727 to_remove.append(i)

1728

1729 [self.records.pop(i) for i in to_remove]

1730

1731 return self.__class__(

1732 self,

1733 self.model,

1734 removed,

1735 )

1736

1737 def sort(self, f: typing.Callable[[T_MetaInstance], Any], reverse: bool = False) -> list[T_MetaInstance]:

1738 """

1739 Returns a list of sorted elements (not sorted in place).

1740 """

1741 return [r for (r, s) in sorted(zip(self.records.values(), self), key=lambda r: f(r[1]), reverse=reverse)]

1742

1743 def __str__(self) -> str:

1744 """

1745 Simple string representation.

1746 """

1747 return f"<TypedRows with {len(self)} records>"

1748

1749 def __repr__(self) -> str:

1750 """

1751 Print a table on repr().

1752 """

1753 data = self.as_dict()

1754 headers = list(next(iter(data.values())).keys())

1755 return mktable(data, headers)

1756

1757 def group_by_value(

1758 self, *fields: "str | Field | TypedField[T]", one_result: bool = False, **kwargs: Any

1759 ) -> dict[T, list[T_MetaInstance]]:

1760 """

1761 Group the rows by a specific field (which will be the dict key).

1762 """

1763 kwargs["one_result"] = one_result

1764 result = super().group_by_value(*fields, **kwargs)

1765 return typing.cast(dict[T, list[T_MetaInstance]], result)

1766

1767 def column(self, column: str = None) -> list[Any]:

1768 """

1769 Get a list of all values in a specific column.

1770

1771 Example:

1772 rows.column('name') -> ['Name 1', 'Name 2', ...]

1773 """

1774 return typing.cast(list[Any], super().column(column))

1775

1776 def as_csv(self) -> str:

1777 """

1778 Dump the data to csv.

1779 """

1780 return typing.cast(str, super().as_csv())

1781

1782 def as_dict(

1783 self,

1784 key: str = None,

1785 compact: bool = False,

1786 storage_to_dict: bool = False,

1787 datetime_to_str: bool = False,

1788 custom_types: list[type] = None,

1789 ) -> dict[int, dict[str, Any]]:

1790 """

1791 Get the data in a dict of dicts.

1792 """

1793 if any([key, compact, storage_to_dict, datetime_to_str, custom_types]):

1794 # functionality not guaranteed

1795 return typing.cast(

1796 dict[int, dict[str, Any]],

1797 super().as_dict(

1798 key or "id",

1799 compact,

1800 storage_to_dict,

1801 datetime_to_str,

1802 custom_types,

1803 ),

1804 )

1805

1806 return {k: v.as_dict() for k, v in self.records.items()}

1807

1808 def as_json(self, mode: str = "object", default: typing.Callable[[Any], Any] = None) -> str:

1809 """

1810 Turn the data into a dict and then dump to JSON.

1811 """

1812 return typing.cast(str, super().as_json(mode=mode, default=default))

1813

1814 def json(self, mode: str = "object", default: typing.Callable[[Any], Any] = None) -> str:

1815 """

1816 Turn the data into a dict and then dump to JSON.

1817 """

1818 return typing.cast(str, super().as_json(mode=mode, default=default))

1819

1820 def as_list(

1821 self,

1822 compact: bool = False,

1823 storage_to_dict: bool = False,

1824 datetime_to_str: bool = False,

1825 custom_types: list[type] = None,

1826 ) -> list[dict[str, Any]]:

1827 """

1828 Get the data in a list of dicts.

1829 """

1830 if any([compact, storage_to_dict, datetime_to_str, custom_types]):

1831 return typing.cast(

1832 list[dict[str, Any]], super().as_list(compact, storage_to_dict, datetime_to_str, custom_types)

1833 )

1834 return [_.as_dict() for _ in self.records.values()]

1835

1836 def __getitem__(self, item: int) -> T_MetaInstance:

1837 """

1838 You can get a specific row by ID from a typedrows by using rows[idx] notation.

1839

1840 Since pydal's implementation differs (they expect a list instead of a dict with id keys),

1841 using rows[0] will return the first row, regardless of its id.

1842 """

1843 try:

1844 return self.records[item]

1845 except KeyError as e:

1846 if item == 0 and (row := self.first()):

1847 # special case: pydal internals think Rows.records is a list, not a dict

1848 return row

1849

1850 raise e

1851

1852 def get(self, item: int) -> typing.Optional[T_MetaInstance]:

1853 """

1854 Get a row by ID, or receive None if it isn't in this result set.

1855 """

1856 return self.records.get(item)

1857

1858 def join(

1859 self,

1860 field: "Field | TypedField[Any]",

1861 name: str = None,

1862 constraint: Query = None,

1863 fields: list[str | Field] = None,

1864 orderby: Optional[str | Field] = None,

1865 ) -> T_MetaInstance:

1866 """

1867 This can be used to JOIN with some relationships after the initial select.

1868

1869 Using the querybuilder's .join() method is prefered!

1870 """

1871 result = super().join(field, name, constraint, fields or [], orderby)

1872 return typing.cast(T_MetaInstance, result)

1873

1874 def export_to_csv_file(

1875 self,

1876 ofile: typing.TextIO,

1877 null: Any = "<NULL>",

1878 delimiter: str = ",",

1879 quotechar: str = '"',

1880 quoting: int = csv.QUOTE_MINIMAL,

1881 represent: bool = False,

1882 colnames: list[str] = None,

1883 write_colnames: bool = True,

1884 *args: Any,

1885 **kwargs: Any,

1886 ) -> None:

1887 """

1888 Shadow export_to_csv_file from Rows, but with typing.

1889

1890 See http://web2py.com/books/default/chapter/29/06/the-database-abstraction-layer?search=export_to_csv_file#Exporting-and-importing-data

1891 """

1892 super().export_to_csv_file(

1893 ofile,

1894 null,

1895 *args,

1896 delimiter=delimiter,

1897 quotechar=quotechar,

1898 quoting=quoting,

1899 represent=represent,

1900 colnames=colnames or self.colnames,

1901 write_colnames=write_colnames,

1902 **kwargs,

1903 )

1904

1905 @classmethod

1906 def from_rows(

1907 cls, rows: Rows, model: typing.Type[T_MetaInstance], metadata: Metadata = None

1908 ) -> "TypedRows[T_MetaInstance]":

1909 """

1910 Internal method to convert a Rows object to a TypedRows.

1911 """

1912 return cls(rows, model, metadata=metadata)

1913

1914 def __json__(self) -> dict[str, Any]:

1915 """

1916 For json-fix.

1917 """

1918 return typing.cast(dict[str, Any], self.as_dict())

1919

1920 def __getstate__(self) -> dict[str, Any]:

1921 """

1922 Used by dill to dump to bytes (exclude db connection etc).

1923 """

1924 return {

1925 "metadata": json.dumps(self.metadata, default=str),

1926 "records": self.records,

1927 }

1928

1929 def __setstate__(self, state: dict[str, Any]) -> None:

1930 """

1931 Used by dill when loading from a bytestring.

1932 """

1933 state["metadata"] = json.loads(state["metadata"])

1934 self.__dict__.update(state)

1935

1936

1937 from .caching import ( # noqa: E402

1938 _remove_cache,

1939 _TypedalCache,

1940 _TypedalCacheDependency,

1941 create_and_hash_cache_key,

1942 get_expire,

1943 load_from_cache,

1944 save_to_cache,

1945)

1946

1947

1948 class QueryBuilder(typing.Generic[T_MetaInstance]):

1949 """

1950 Abstration on top of pydal's query system.

1951 """

1952

1953 model: typing.Type[T_MetaInstance]

1954 query: Query

1955 select_args: list[Any]

1956 select_kwargs: dict[str, Any]

1957 relationships: dict[str, Relationship[Any]]

1958 metadata: Metadata

1959

1960 def __init__(

1961 self,

1962 model: typing.Type[T_MetaInstance],

1963 add_query: Optional[Query] = None,

1964 select_args: Optional[list[Any]] = None,

1965 select_kwargs: Optional[dict[str, Any]] = None,

1966 relationships: dict[str, Relationship[Any]] = None,

1967 metadata: Metadata = None,

1968 ):

1969 """

1970 Normally, you wouldn't manually initialize a QueryBuilder but start using a method on a TypedTable.

1971

1972 Example:

1973 MyTable.where(...) -> QueryBuilder[MyTable]

1974 """

1975 self.model = model

1976 table = model._ensure_table_defined()

1977 default_query = typing.cast(Query, table.id > 0)

1978 self.query = add_query or default_query

1979 self.select_args = select_args or []

1980 self.select_kwargs = select_kwargs or {}

1981 self.relationships = relationships or {}

1982 self.metadata = metadata or {}

1983

1984 def __str__(self) -> str:

1985 """

1986 Simple string representation for the query builder.

1987 """

1988 return f"QueryBuilder for {self.model}"

1989

1990 def __repr__(self) -> str:

1991 """

1992 Advanced string representation for the query builder.

1993 """

1994 return (

1995 f"<QueryBuilder for {self.model} with "

1996 f"{len(self.select_args)} select args; "

1997 f"{len(self.select_kwargs)} select kwargs; "

1998 f"{len(self.relationships)} relationships; "

1999 f"query: {bool(self.query)}; "

2000 f"metadata: {self.metadata}; "

2001 f">"

2002 )

2003

2004 def __bool__(self) -> bool:

2005 """

2006 Querybuilder is truthy if it has rows.

2007 """

2008 return self.count() > 0

2009

2010 def _extend(

2011 self,

2012 add_query: Optional[Query] = None,

2013 overwrite_query: Optional[Query] = None,

2014 select_args: Optional[list[Any]] = None,

2015 select_kwargs: Optional[dict[str, Any]] = None,

2016 relationships: dict[str, Relationship[Any]] = None,

2017 metadata: Metadata = None,

2018 ) -> "QueryBuilder[T_MetaInstance]":

2019 return QueryBuilder(

2020 self.model,

2021 (add_query & self.query) if add_query else overwrite_query or self.query,

2022 (self.select_args + select_args) if select_args else self.select_args,

2023 (self.select_kwargs | select_kwargs) if select_kwargs else self.select_kwargs,

2024 (self.relationships | relationships) if relationships else self.relationships,

2025 (self.metadata | (metadata or {})) if metadata else self.metadata,

2026 )

2027

2028 def select(self, *fields: Any, **options: Any) -> "QueryBuilder[T_MetaInstance]":

2029 """

2030 Fields: database columns by name ('id'), by field reference (table.id) or other (e.g. table.ALL).

2031

2032 Options:

2033 paraphrased from the web2py pydal docs,

2034 For more info, see http://www.web2py.com/books/default/chapter/29/06/the-database-abstraction-layer#orderby-groupby-limitby-distinct-having-orderby_on_limitby-join-left-cache

2035

2036 orderby: field(s) to order by. Supported:

2037 table.name - sort by name, ascending

2038 ~table.name - sort by name, descending

2039 <random> - sort randomly

2040 table.name|table.id - sort by two fields (first name, then id)

2041

2042 groupby, having: together with orderby:

2043 groupby can be a field (e.g. table.name) to group records by

2044 having can be a query, only those `having` the condition are grouped

2045

2046 limitby: tuple of min and max. When using the query builder, .paginate(limit, page) is recommended.

2047 distinct: bool/field. Only select rows that differ

2048 orderby_on_limitby (bool, default: True): by default, an implicit orderby is added when doing limitby.

2049 join: othertable.on(query) - do an INNER JOIN. Using TypeDAL relationships with .join() is recommended!

2050 left: othertable.on(query) - do a LEFT JOIN. Using TypeDAL relationships with .join() is recommended!

2051 cache: cache the query result to speed up repeated queries; e.g. (cache=(cache.ram, 3600), cacheable=True)

2052 """

2053 return self._extend(select_args=list(fields), select_kwargs=options)

2054

2055 def where(

2056 self,

2057 *queries_or_lambdas: Query | typing.Callable[[typing.Type[T_MetaInstance]], Query],

2058 **filters: Any,

2059 ) -> "QueryBuilder[T_MetaInstance]":

2060 """

2061 Extend the builder's query.

2062

2063 Can be used in multiple ways:

2064 .where(Query) -> with a direct query such as `Table.id == 5`

2065 .where(lambda table: table.id == 5) -> with a query via a lambda

2066 .where(id=5) -> via keyword arguments

2067

2068 When using multiple where's, they will be ANDed:

2069 .where(lambda table: table.id == 5).where(lambda table: table.id == 6) == (table.id == 5) & (table.id=6)

2070 When passing multiple queries to a single .where, they will be ORed:

2071 .where(lambda table: table.id == 5, lambda table: table.id == 6) == (table.id == 5) | (table.id=6)

2072 """

2073 new_query = self.query

2074 table = self.model._ensure_table_defined()

2075

2076 for field, value in filters.items():

2077 new_query &= table[field] == value

2078

2079 subquery: DummyQuery | Query = DummyQuery()

2080 for query_or_lambda in queries_or_lambdas:

2081 if isinstance(query_or_lambda, _Query):

2082 subquery |= typing.cast(Query, query_or_lambda)

2083 elif callable(query_or_lambda):

2084 if result := query_or_lambda(self.model):

2085 subquery |= result

2086 elif isinstance(query_or_lambda, (Field, _Field)) or is_typed_field(query_or_lambda):

2087 subquery |= typing.cast(Query, query_or_lambda != None)

2088 else:

2089 raise ValueError(f"Unexpected query type ({type(query_or_lambda)}).")

2090

2091 if subquery:

2092 new_query &= subquery

2093

2094 return self._extend(overwrite_query=new_query)

2095

2096 def join(

2097 self,

2098 *fields: str | typing.Type[TypedTable],

2099 method: JOIN_OPTIONS = None,

2100 on: OnQuery | list[Expression] | Expression = None,

2101 condition: Condition = None,

2102 ) -> "QueryBuilder[T_MetaInstance]":

2103 """

2104 Include relationship fields in the result.

2105

2106 `fields` can be names of Relationships on the current model.

2107 If no fields are passed, all will be used.

2108

2109 By default, the `method` defined in the relationship is used.

2110 This can be overwritten with the `method` keyword argument (left or inner)

2111 """

2112 # todo: allow limiting amount of related rows returned for join?

2113

2114 relationships = self.model.get_relationships()

2115

2116 if condition and on:

2117 raise ValueError("condition and on can not be used together!")

2118 elif condition:

2119 if len(fields) != 1:

2120 raise ValueError("join(field, condition=...) can only be used with exactly one field!")

2121

2122 if isinstance(condition, pydal.objects.Query):

2123 condition = as_lambda(condition)

2124

2125 relationships = {str(fields[0]): relationship(fields[0], condition=condition, join=method)}

2126 elif on:

2127 if len(fields) != 1:

2128 raise ValueError("join(field, on=...) can only be used with exactly one field!")

2129

2130 if isinstance(on, pydal.objects.Expression):

2131 on = [on]

2132

2133 if isinstance(on, list):

2134 on = as_lambda(on)

2135 relationships = {str(fields[0]): relationship(fields[0], on=on, join=method)}

2136

2137 else:

2138 if fields:

2139 # join on every relationship

2140 relationships = {str(k): relationships[str(k)] for k in fields}

2141

2142 if method:

2143 relationships = {str(k): r.clone(join=method) for k, r in relationships.items()}

2144

2145 return self._extend(relationships=relationships)

2146

2147 def cache(

2148 self, *deps: Any, expires_at: Optional[dt.datetime] = None, ttl: Optional[int | dt.timedelta] = None

2149 ) -> "QueryBuilder[T_MetaInstance]":

2150 """

2151 Enable caching for this query to load repeated calls from a dill row \

2152 instead of executing the sql and collecing matching rows again.

2153 """

2154 existing = self.metadata.get("cache", {})

2155

2156 metadata: Metadata = {}

2157

2158 cache_meta = typing.cast(

2159 CacheMetadata,

2160 self.metadata.get("cache", {})

2161 | {

2162 "enabled": True,

2163 "depends_on": existing.get("depends_on", []) + [str(_) for _ in deps],

2164 "expires_at": get_expire(expires_at=expires_at, ttl=ttl),

2165 },

2166 )

2167

2168 metadata["cache"] = cache_meta

2169 return self._extend(metadata=metadata)

2170

2171 def _get_db(self) -> TypeDAL:

2172 if db := self.model._db:

2173 return db

2174 else: # pragma: no cover

2175 raise EnvironmentError("@define or db.define is not called on this class yet!")

2176

2177 def _select_arg_convert(self, arg: Any) -> Any:

2178 # typedfield are not really used at runtime anymore, but leave it in for safety:

2179 if isinstance(arg, TypedField): # pragma: no cover

2180 arg = arg._field

2181

2182 return arg

2183

2184 def delete(self) -> list[int]:

2185 """

2186 Based on the current query, delete rows and return a list of deleted IDs.

2187 """

2188 db = self._get_db()

2189 removed_ids = [_.id for _ in db(self.query).select("id")]

2190 if db(self.query).delete():

2191 # success!

2192 return removed_ids

2193

2194 return []

2195

2196 def _delete(self) -> str:

2197 db = self._get_db()

2198 return str(db(self.query)._delete())

2199

2200 def update(self, **fields: Any) -> list[int]:

2201 """

2202 Based on the current query, update `fields` and return a list of updated IDs.

2203 """

2204 # todo: limit?

2205 db = self._get_db()

2206 updated_ids = db(self.query).select("id").column("id")

2207 if db(self.query).update(**fields):

2208 # success!

2209 return updated_ids

2210

2211 return []

2212

2213 def _update(self, **fields: Any) -> str:

2214 db = self._get_db()

2215 return str(db(self.query)._update(**fields))

2216

2217 def _before_query(self, mut_metadata: Metadata, add_id: bool = True) -> tuple[Query, list[Any], dict[str, Any]]:

2218 select_args = [self._select_arg_convert(_) for _ in self.select_args] or [self.model.ALL]

2219 select_kwargs = self.select_kwargs.copy()

2220 query = self.query

2221 model = self.model

2222 mut_metadata["query"] = query

2223 # require at least id of main table:

2224 select_fields = ", ".join([str(_) for _ in select_args])

2225 tablename = str(model)

2226

2227 if add_id and f"{tablename}.id" not in select_fields:

2228 # fields of other selected, but required ID is missing.

2229 select_args.append(model.id)

2230

2231 if self.relationships:

2232 query, select_args = self._handle_relationships_pre_select(query, select_args, select_kwargs, mut_metadata)

2233

2234 return query, select_args, select_kwargs

2235

2236 def to_sql(self, add_id: bool = False) -> str:

2237 """

2238 Generate the SQL for the built query.

2239 """

2240 db = self._get_db()

2241

2242 query, select_args, select_kwargs = self._before_query({}, add_id=add_id)

2243

2244 return str(db(query)._select(*select_args, **select_kwargs))

2245

2246 def _collect(self) -> str:

2247 """

2248 Alias for to_sql, pydal-like syntax.

2249 """

2250 return self.to_sql()

2251

2252 def _collect_cached(self, metadata: Metadata) -> "TypedRows[T_MetaInstance] | None":

2253 expires_at = metadata["cache"].get("expires_at")

2254 metadata["cache"] |= {

2255 # key is partly dependant on cache metadata but not these:

2256 "key": None,

2257 "status": None,

2258 "cached_at": None,

2259 "expires_at": None,

2260 } # type: ignore

2261

2262 _, key = create_and_hash_cache_key(

2263 self.model,

2264 metadata,

2265 self.query,

2266 self.select_args,

2267 self.select_kwargs,

2268 self.relationships.keys(),

2269 )

2270

2271 # re-set after creating key:

2272 metadata["cache"]["expires_at"] = expires_at

2273 metadata["cache"]["key"] = key

2274

2275 return load_from_cache(key)

2276

2277 def collect(

2278 self, verbose: bool = False, _to: typing.Type["TypedRows[Any]"] = None, add_id: bool = True

2279 ) -> "TypedRows[T_MetaInstance]":

2280 """

2281 Execute the built query and turn it into model instances, while handling relationships.

2282 """

2283 if _to is None:

2284 _to = TypedRows

2285

2286 db = self._get_db()

2287 metadata = typing.cast(Metadata, self.metadata.copy())

2288

2289 if metadata.get("cache", {}).get("enabled") and (result := self._collect_cached(metadata)):

2290 return result

2291

2292 query, select_args, select_kwargs = self._before_query(metadata, add_id=add_id)

2293

2294 metadata["sql"] = db(query)._select(*select_args, **select_kwargs)

2295

2296 if verbose: # pragma: no cover

2297 print(metadata["sql"])

2298

2299 rows: Rows = db(query).select(*select_args, **select_kwargs)

2300

2301 metadata["final_query"] = str(query)

2302 metadata["final_args"] = [str(_) for _ in select_args]

2303 metadata["final_kwargs"] = select_kwargs

2304

2305 if verbose: # pragma: no cover

2306 print(rows)

2307

2308 if not self.relationships:

2309 # easy

2310 typed_rows = _to.from_rows(rows, self.model, metadata=metadata)

2311

2312 else:

2313 # harder: try to match rows to the belonging objects

2314 # assume structure of {'table': <data>} per row.

2315 # if that's not the case, return default behavior again

2316 typed_rows = self._collect_with_relationships(rows, metadata=metadata, _to=_to)

2317

2318 # only saves if requested in metadata:

2319 return save_to_cache(typed_rows, rows)

2320

2321 def _handle_relationships_pre_select(

2322 self,

2323 query: Query,

2324 select_args: list[Any],

2325 select_kwargs: dict[str, Any],

2326 metadata: Metadata,

2327 ) -> tuple[Query, list[Any]]:

2328 db = self._get_db()

2329 model = self.model

2330

2331 metadata["relationships"] = set(self.relationships.keys())

2332

2333 # query = self._update_query_for_inner(db, model, query)

2334 join = []

2335 for key, relation in self.relationships.items():

2336 if not relation.condition or relation.join != "inner":

2337 continue

2338

2339 other = relation.get_table(db)

2340 other = other.with_alias(f"{key}_{hash(relation)}")

2341 join.append(other.on(relation.condition(model, other)))

2342

2343 if limitby := select_kwargs.pop("limitby", None):

2344 # if limitby + relationships:

2345 # 1. get IDs of main table entries that match 'query'

2346 # 2. change query to .belongs(id)

2347 # 3. add joins etc

2348

2349 kwargs = {"limitby": limitby}

2350

2351 if join:

2352 kwargs["join"] = join

2353

2354 ids = db(query)._select(model.id, **kwargs)

2355 query = model.id.belongs(ids)

2356 metadata["ids"] = ids

2357

2358 if join:

2359 select_kwargs["join"] = join

2360

2361 left = []

2362

2363 for key, relation in self.relationships.items():

2364 other = relation.get_table(db)

2365 method: JOIN_OPTIONS = relation.join or DEFAULT_JOIN_OPTION

2366

2367 select_fields = ", ".join([str(_) for _ in select_args])

2368 pre_alias = str(other)

2369

2370 if f"{other}." not in select_fields:

2371 # no fields of other selected. add .ALL:

2372 select_args.append(other.ALL)

2373 elif f"{other}.id" not in select_fields:

2374 # fields of other selected, but required ID is missing.

2375 select_args.append(other.id)

2376

2377 if relation.on:

2378 # if it has a .on, it's always a left join!

2379 on = relation.on(model, other)

2380 if not isinstance(on, list): # pragma: no cover

2381 on = [on]

2382

2383 left.extend(on)

2384 elif method == "left":

2385 # .on not given, generate it:

2386 other = other.with_alias(f"{key}_{hash(relation)}")

2387 condition = typing.cast(Query, relation.condition(model, other))

2388 left.append(other.on(condition))

2389 else:

2390 # else: inner join (handled earlier)

2391 other = other.with_alias(f"{key}_{hash(relation)}") # only for replace

2392 # other = other.with_alias(f"{key}_{hash(relation)}")

2393 # query &= relation.condition(model, other)

2394

2395 # if no fields of 'other' are included, add other.ALL

2396 # else: only add other.id if missing

2397 select_fields = ", ".join([str(_) for _ in select_args])

2398

2399 post_alias = str(other).split(" AS ")[-1]

2400 if pre_alias != post_alias:

2401 # replace .select's with aliased:

2402 select_fields = select_fields.replace(

2403 f"{pre_alias}.",

2404 f"{post_alias}.",

2405 )

2406

2407 select_args = select_fields.split(", ")

2408

2409 select_kwargs["left"] = left

2410 return query, select_args

2411

2412 def _collect_with_relationships(

2413 self, rows: Rows, metadata: Metadata, _to: typing.Type["TypedRows[Any]"] = None

2414 ) -> "TypedRows[T_MetaInstance]":

2415 """

2416 Transform the raw rows into Typed Table model instances.

2417 """

2418 db = self._get_db()

2419 main_table = self.model._ensure_table_defined()

2420

2421 records = {}

2422 seen_relations: dict[str, set[str]] = defaultdict(set) # main id -> set of col + id for relation

2423

2424 for row in rows:

2425 main = row[main_table]

2426 main_id = main.id

2427

2428 if main_id not in records:

2429 records[main_id] = self.model(main)

2430 records[main_id]._with = list(self.relationships.keys())

2431

2432 # setup up all relationship defaults (once)

2433 for col, relationship in self.relationships.items():

2434 records[main_id][col] = [] if relationship.multiple else None

2435

2436 # now add other relationship data

2437 for column, relation in self.relationships.items():

2438 relationship_column = f"{column}_{hash(relation)}"

2439

2440 # relationship_column works for aliases with the same target column.

2441 # if col + relationship not in the row, just use the regular name.

2442

2443 relation_data = (

2444 row[relationship_column] if relationship_column in row else row[relation.get_table_name()]

2445 )

2446

2447 if relation_data.id is None:

2448 # always skip None ids

2449 continue

2450

2451 if f"{column}-{relation_data.id}" in seen_relations[main_id]:

2452 # speed up duplicates

2453 continue

2454 else:

2455 seen_relations[main_id].add(f"{column}-{relation_data.id}")

2456

2457 relation_table = relation.get_table(db)

2458 # hopefully an instance of a typed table and a regular row otherwise:

2459 instance = relation_table(relation_data) if looks_like(relation_table, TypedTable) else relation_data

2460

2461 if relation.multiple:

2462 # create list of T

2463 if not isinstance(records[main_id].get(column), list): # pragma: no cover

2464 # should already be set up before!

2465 setattr(records[main_id], column, [])

2466

2467 records[main_id][column].append(instance)

2468 else:

2469 # create single T

2470 records[main_id][column] = instance

2471

2472 return _to(rows, self.model, records, metadata=metadata)

2473

2474 def collect_or_fail(self, exception: Exception = None) -> "TypedRows[T_MetaInstance]":

2475 """

2476 Call .collect() and raise an error if nothing found.

2477

2478 Basically unwraps Optional type.

2479 """

2480 if result := self.collect():

2481 return result

2482

2483 if not exception:

2484 exception = ValueError("Nothing found!")

2485

2486 raise exception

2487

2488 def __iter__(self) -> typing.Generator[T_MetaInstance, None, None]:

2489 """

2490 You can start iterating a Query Builder object before calling collect, for ease of use.

2491 """

2492 yield from self.collect()

2493

2494 def count(self) -> int:

2495 """

2496 Return the amount of rows matching the current query.

2497 """

2498 db = self._get_db()

2499 model = self.model

2500 query = self.query

2501

2502 for key, relation in self.relationships.items():

2503 if not relation.condition or relation.join != "inner":

2504 continue

2505

2506 other = relation.get_table(db)

2507 other = other.with_alias(f"{key}_{hash(relation)}")

2508 query &= relation.condition(model, other)

2509

2510 return db(query).count()

2511

2512 def __paginate(

2513 self,

2514 limit: int,

2515 page: int = 1,

2516 ) -> "QueryBuilder[T_MetaInstance]":

2517 _from = limit * (page - 1)

2518 _to = limit * page

2519

2520 available = self.count()

2521

2522 metadata: Metadata = {}

2523

2524 metadata["pagination"] = {

2525 "limit": limit,

2526 "current_page": page,

2527 "max_page": math.ceil(available / limit),

2528 "rows": available,

2529 "min_max": (_from, _to),

2530 }

2531

2532 return self._extend(select_kwargs={"limitby": (_from, _to)}, metadata=metadata)

2533

2534 def paginate(self, limit: int, page: int = 1, verbose: bool = False) -> "PaginatedRows[T_MetaInstance]":

2535 """

2536 Paginate transforms the more readable `page` and `limit` to pydals internal limit and offset.

2537

2538 Note: when using relationships, this limit is only applied to the 'main' table and any number of extra rows \

2539 can be loaded with relationship data!

2540 """

2541 builder = self.__paginate(limit, page)

2542

2543 rows = typing.cast(PaginatedRows[T_MetaInstance], builder.collect(verbose=verbose, _to=PaginatedRows))

2544

2545 rows._query_builder = builder

2546 return rows

2547

2548 def _paginate(

2549 self,

2550 limit: int,

2551 page: int = 1,

2552 ) -> str:

2553 builder = self.__paginate(limit, page)

2554 return builder._collect()

2555

2556 def chunk(self, chunk_size: int) -> typing.Generator["TypedRows[T_MetaInstance]", Any, None]:

2557 """

2558 Generator that yields rows from a paginated source in chunks.

2559

2560 This function retrieves rows from a paginated data source in chunks of the

2561 specified `chunk_size` and yields them as TypedRows.

2562

2563 Example:

2564 ```

2565 for chunk_of_rows in Table.where(SomeTable.id > 5).chunk(100):

2566 for row in chunk_of_rows:

2567 # Process each row within the chunk.

2568 pass

2569 ```

2570 """

2571 page = 1

2572

2573 while rows := self.__paginate(chunk_size, page).collect():

2574 yield rows

2575 page += 1

2576

2577 def first(self, verbose: bool = False) -> T_MetaInstance | None:

2578 """

2579 Get the first row matching the currently built query.

2580

2581 Also adds paginate, since it would be a waste to select more rows than needed.

2582 """

2583 if row := self.paginate(page=1, limit=1, verbose=verbose).first():

2584 return self.model.from_row(row)

2585 else:

2586 return None

2587

2588 def _first(self) -> str:

2589 return self._paginate(page=1, limit=1)

2590

2591 def first_or_fail(self, exception: Exception = None, verbose: bool = False) -> T_MetaInstance:

2592 """

2593 Call .first() and raise an error if nothing found.

2594

2595 Basically unwraps Optional type.

2596 """

2597 if inst := self.first(verbose=verbose):

2598 return inst

2599

2600 if not exception:

2601 exception = ValueError("Nothing found!")

2602

2603 raise exception

2604

2605

2606 S = typing.TypeVar("S")

2607

2608

2609 class PaginatedRows(TypedRows[T_MetaInstance]):

2610 """

2611 Extension on top of rows that is used when calling .paginate() instead of .collect().

2612 """

2613

2614 _query_builder: QueryBuilder[T_MetaInstance]

2615

2616 @property

2617 def data(self) -> list[T_MetaInstance]:

2618 """

2619 Get the underlying data.

2620 """

2621 return list(self.records.values())

2622

2623 @property

2624 def pagination(self) -> Pagination:

2625 """

2626 Get all page info.

2627 """

2628 pagination_data = self.metadata["pagination"]

2629

2630 has_next_page = pagination_data["current_page"] < pagination_data["max_page"]

2631 has_prev_page = pagination_data["current_page"] > 1

2632 return {

2633 "total_items": pagination_data["rows"],

2634 "current_page": pagination_data["current_page"],

2635 "per_page": pagination_data["limit"],

2636 "total_pages": pagination_data["max_page"],

2637 "has_next_page": has_next_page,

2638 "has_prev_page": has_prev_page,

2639 "next_page": pagination_data["current_page"] + 1 if has_next_page else None,

2640 "prev_page": pagination_data["current_page"] - 1 if has_prev_page else None,

2641 }

2642

2643 def next(self) -> Self: # noqa: A003

2644 """

2645 Get the next page.

2646 """

2647 data = self.metadata["pagination"]

2648 if data["current_page"] >= data["max_page"]:

2649 raise StopIteration("Final Page")

2650

2651 return self._query_builder.paginate(limit=data["limit"], page=data["current_page"] + 1)

2652

2653 def previous(self) -> Self:

2654 """

2655 Get the previous page.

2656 """

2657 data = self.metadata["pagination"]

2658 if data["current_page"] <= 1:

2659 raise StopIteration("First Page")

2660

2661 return self._query_builder.paginate(limit=data["limit"], page=data["current_page"] - 1)

2662

2663 def as_dict(self, *_: Any, **__: Any) -> PaginateDict: # type: ignore

2664 """

2665 Convert to a dictionary with pagination info and original data.

2666

2667 All arguments are ignored!

2668 """

2669 return {"data": super().as_dict(), "pagination": self.pagination}

2670

2671

2672 class TypedSet(pydal.objects.Set): # type: ignore # pragma: no cover

2673 """

2674 Used to make pydal Set more typed.

2675

2676 This class is not actually used, only 'cast' by TypeDAL.__call__

2677 """

2678

2679 def count(self, distinct: bool = None, cache: dict[str, Any] = None) -> int:

2680 """

2681 Count returns an int.

2682 """

2683 result = super().count(distinct, cache)

2684 return typing.cast(int, result)

2685

2686 def select(self, *fields: Any, **attributes: Any) -> TypedRows[T_MetaInstance]:

2687 """

2688 Select returns a TypedRows of a user defined table.

2689

2690 Example:

2691 result: TypedRows[MyTable] = db(MyTable.id > 0).select()

2692

2693 for row in result:

2694 typing.reveal_type(row) # MyTable

2695 """

2696 rows = super().select(*fields, **attributes)

2697 return typing.cast(TypedRows[T_MetaInstance], rows)