Coverage for src/typedal/core.py: 100%

1"""

2 Core functionality of TypeDAL.

3"""

4

5 import contextlib

6 import csv

7 import datetime as dt

8 import inspect

9 import json

10 import math

11 import types

12 import typing

13 import warnings

14 from collections import defaultdict

15 from copy import copy

16 from decimal import Decimal

17 from pathlib import Path

18 from typing import Any, Optional

19

20 import pydal

21 from pydal._globals import DEFAULT

22 from pydal.objects import Field as _Field

23 from pydal.objects import Query as _Query

24 from pydal.objects import Row

25 from pydal.objects import Table as _Table

26 from typing_extensions import Self

27

28 from .config import TypeDALConfig, load_config

29 from .helpers import (

30 DummyQuery,

31 all_annotations,

32 all_dict,

33 as_lambda,

34 extract_type_optional,

35 filter_out,

36 instanciate,

37 is_union,

38 looks_like,

39 mktable,

40 origin_is_subclass,

41 to_snake,

42 unwrap_type,

43)

44 from .serializers import as_json

45 from .types import (

46 AfterDeleteCallable,

47 AfterInsertCallable,

48 AfterUpdateCallable,

49 AnyDict,

50 BeforeDeleteCallable,

51 BeforeInsertCallable,

52 BeforeUpdateCallable,

53 CacheMetadata,

54 Expression,

55 Field,

56 Metadata,

57 PaginateDict,

58 Pagination,

59 Query,

60 Rows,

61 Validator,

62 _Types,

63)

64

65# use typing.cast(type, ...) to make mypy happy with unions

66 T_annotation = typing.Type[Any] | types.UnionType

67 T_Query = typing.Union["Table", Query, bool, None, "TypedTable", typing.Type["TypedTable"]]

68 T_Value = typing.TypeVar("T_Value") # actual type of the Field (via Generic)

69 T_MetaInstance = typing.TypeVar("T_MetaInstance", bound="TypedTable") # bound="TypedTable"; bound="TableMeta"

70 T = typing.TypeVar("T")

71

72 BASIC_MAPPINGS: dict[T_annotation, str] = {

73 str: "string",

74 int: "integer",

75 bool: "boolean",

76 bytes: "blob",

77 float: "double",

78 object: "json",

79 Decimal: "decimal(10,2)",

80 dt.date: "date",

81 dt.time: "time",

82 dt.datetime: "datetime",

83}

84

85

86 def is_typed_field(cls: Any) -> typing.TypeGuard["TypedField[Any]"]:

87 """

88 Is `cls` an instance or subclass of TypedField?

89

90 Deprecated

91 """

92 return (

93 isinstance(cls, TypedField)

94 or isinstance(typing.get_origin(cls), type)

95 and issubclass(typing.get_origin(cls), TypedField)

96 )

97

98

99 JOIN_OPTIONS = typing.Literal["left", "inner", None]

100 DEFAULT_JOIN_OPTION: JOIN_OPTIONS = "left"

101

102# table-ish paramter:

103 P_Table = typing.Union[typing.Type["TypedTable"], pydal.objects.Table]

104

105 Condition: typing.TypeAlias = typing.Optional[

106 typing.Callable[

107 # self, other -> Query

108 [P_Table, P_Table],

109 Query | bool,

110 ]

111]

112

113 OnQuery: typing.TypeAlias = typing.Optional[

114 typing.Callable[

115 # self, other -> list of .on statements

116 [P_Table, P_Table],

117 list[Expression],

118 ]

119]

120

121 To_Type = typing.TypeVar("To_Type", type[Any], typing.Type[Any], str)

122

123

124 class Relationship(typing.Generic[To_Type]):

125 """

126 Define a relationship to another table.

127 """

128

129 _type: To_Type

130 table: typing.Type["TypedTable"] | type | str

131 condition: Condition

132 on: OnQuery

133 multiple: bool

134 join: JOIN_OPTIONS

135

136 def __init__(

137 self,

138 _type: To_Type,

139 condition: Condition = None,

140 join: JOIN_OPTIONS = None,

141 on: OnQuery = None,

142 ):

143 """

144 Should not be called directly, use relationship() instead!

145 """

146 if condition and on:

147 warnings.warn(f"Relation | Both specified! {condition=} {on=} {_type=}")

148 raise ValueError("Please specify either a condition or an 'on' statement for this relationship!")

149

150 self._type = _type

151 self.condition = condition

152 self.join = "left" if on else join # .on is always left join!

153 self.on = on

154

155 if args := typing.get_args(_type):

156 self.table = unwrap_type(args[0])

157 self.multiple = True

158 else:

159 self.table = _type

160 self.multiple = False

161

162 if isinstance(self.table, str):

163 self.table = TypeDAL.to_snake(self.table)

164

165 def clone(self, **update: Any) -> "Relationship[To_Type]":

166 """

167 Create a copy of the relationship, possibly updated.

168 """

169 return self.__class__(

170 update.get("_type") or self._type,

171 update.get("condition") or self.condition,

172 update.get("join") or self.join,

173 update.get("on") or self.on,

174 )

175

176 def __repr__(self) -> str:

177 """

178 Representation of the relationship.

179 """

180 if callback := self.condition or self.on:

181 src_code = inspect.getsource(callback).strip()

182 else:

183 cls_name = self._type if isinstance(self._type, str) else self._type.__name__ # type: ignore

184 src_code = f"to {cls_name} (missing condition)"

185

186 join = f":{self.join}" if self.join else ""

187 return f"<Relationship{join} {src_code}>"

188

189 def get_table(self, db: "TypeDAL") -> typing.Type["TypedTable"]:

190 """

191 Get the table this relationship is bound to.

192 """

193 table = self.table # can be a string because db wasn't available yet

194 if isinstance(table, str):

195 if mapped := db._class_map.get(table):

196 # yay

197 return mapped

198

199 # boo, fall back to untyped table but pretend it is typed:

200 return typing.cast(typing.Type["TypedTable"], db[table]) # eh close enough!

201

202 return table

203

204 def get_table_name(self) -> str:

205 """

206 Get the name of the table this relationship is bound to.

207 """

208 if isinstance(self.table, str):

209 return self.table

210

211 if isinstance(self.table, pydal.objects.Table):

212 return str(self.table)

213

214 # else: typed table

215 try:

216 table = self.table._ensure_table_defined() if issubclass(self.table, TypedTable) else self.table

217 except Exception: # pragma: no cover

218 table = self.table

219

220 return str(table)

221

222 def __get__(self, instance: Any, owner: Any) -> typing.Optional[list[Any]] | "Relationship[To_Type]":

223 """

224 Relationship is a descriptor class, which can be returned from a class but not an instance.

225

226 For an instance, using .join() will replace the Relationship with the actual data.

227 If you forgot to join, a warning will be shown and empty data will be returned.

228 """

229 if not instance:

230 # relationship queried on class, that's allowed

231 return self

232

233 warnings.warn(

234 "Trying to get data from a relationship object! Did you forget to join it?", category=RuntimeWarning

235 )

236 if self.multiple:

237 return []

238 else:

239 return None

240

241

242 def relationship(

243 _type: To_Type, condition: Condition = None, join: JOIN_OPTIONS = None, on: OnQuery = None

244) -> Relationship[To_Type]:

245 """

246 Define a relationship to another table, when its id is not stored in the current table.

247

248 Example:

249 class User(TypedTable):

250 name: str

251

252 posts = relationship(list["Post"], condition=lambda self, post: self.id == post.author, join='left')

253

254 class Post(TypedTable):

255 title: str

256 author: User

257

258 User.join("posts").first() # User instance with list[Post] in .posts

259

260 Here, Post stores the User ID, but `relationship(list["Post"])` still allows you to get the user's posts.

261 In this case, the join strategy is set to LEFT so users without posts are also still selected.

262

263 For complex queries with a pivot table, a `on` can be set insteaad of `condition`:

264 class User(TypedTable):

265 ...

266

267 tags = relationship(list["Tag"], on=lambda self, tag: [

268 Tagged.on(Tagged.entity == entity.gid),

269 Tag.on((Tagged.tag == tag.id)),

270 ])

271

272 If you'd try to capture this in a single 'condition', pydal would create a cross join which is much less efficient.

273 """

274 return Relationship(_type, condition, join, on)

275

276

277 def _generate_relationship_condition(

278 _: typing.Type["TypedTable"], key: str, field: typing.Union["TypedField[Any]", "Table", typing.Type["TypedTable"]]

279) -> Condition:

280 origin = typing.get_origin(field)

281 # else: generic

282

283 if origin == list:

284 # field = typing.get_args(field)[0] # actual field

285 # return lambda _self, _other: cls[key].contains(field)

286

287 return lambda _self, _other: _self[key].contains(_other.id)

288 else:

289 # normal reference

290 # return lambda _self, _other: cls[key] == field.id

291 return lambda _self, _other: _self[key] == _other.id

292

293

294 def to_relationship(

295 cls: typing.Type["TypedTable"] | type[Any],

296 key: str,

297 field: typing.Union["TypedField[Any]", "Table", typing.Type["TypedTable"]],

298) -> typing.Optional[Relationship[Any]]:

299 """

300 Used to automatically create relationship instance for reference fields.

301

302 Example:

303 class MyTable(TypedTable):

304 reference: OtherTable

305

306 `reference` contains the id of an Other Table row.

307 MyTable.relationships should have 'reference' as a relationship, so `MyTable.join('reference')` should work.

308

309 This function will automatically perform this logic (called in db.define):

310 to_relationship(MyTable, 'reference', OtherTable) -> Relationship[OtherTable]

311

312 Also works for list:reference (list[OtherTable]) and TypedField[OtherTable].

313 """

314 if looks_like(field, TypedField):

315 if args := typing.get_args(field):

316 field = args[0]

317 else:

318 # weird

319 return None

320

321 field, optional = extract_type_optional(field)

322

323 try:

324 condition = _generate_relationship_condition(cls, key, field)

325 except Exception as e: # pragma: no cover

326 warnings.warn("Could not generate Relationship condition", source=e)

327 condition = None

328

329 if not condition: # pragma: no cover

330 # something went wrong, not a valid relationship

331 warnings.warn(f"Invalid relationship for {cls.__name__}.{key}: {field}")

332 return None

333

334 join = "left" if optional or typing.get_origin(field) == list else "inner"

335

336 return Relationship(typing.cast(type[TypedTable], field), condition, typing.cast(JOIN_OPTIONS, join))

337

338

339 class TypeDAL(pydal.DAL): # type: ignore

340 """

341 Drop-in replacement for pyDAL with layer to convert class-based table definitions to classical pydal define_tables.

342 """

343

344 _config: TypeDALConfig

345

346 def __init__(

347 self,

348 uri: Optional[str] = None, # default from config or 'sqlite:memory'

349 pool_size: int = None, # default 1 if sqlite else 3

350 folder: Optional[str | Path] = None, # default 'databases' in config

351 db_codec: str = "UTF-8",

352 check_reserved: Optional[list[str]] = None,

353 migrate: Optional[bool] = None, # default True by config

354 fake_migrate: Optional[bool] = None, # default False by config

355 migrate_enabled: bool = True,

356 fake_migrate_all: bool = False,

357 decode_credentials: bool = False,

358 driver_args: Optional[AnyDict] = None,

359 adapter_args: Optional[AnyDict] = None,

360 attempts: int = 5,

361 auto_import: bool = False,

362 bigint_id: bool = False,

363 debug: bool = False,

364 lazy_tables: bool = False,

365 db_uid: Optional[str] = None,

366 after_connection: typing.Callable[..., Any] = None,

367 tables: Optional[list[str]] = None,

368 ignore_field_case: bool = True,

369 entity_quoting: bool = True,

370 table_hash: Optional[str] = None,

371 enable_typedal_caching: bool = None,

372 use_pyproject: bool | str = True,

373 use_env: bool | str = True,

374 connection: Optional[str] = None,

375 config: Optional[TypeDALConfig] = None,

376 ) -> None:

377 """

378 Adds some internal tables after calling pydal's default init.

379

380 Set enable_typedal_caching to False to disable this behavior.

381 """

382 config = config or load_config(connection, _use_pyproject=use_pyproject, _use_env=use_env)

383 config.update(

384 database=uri,

385 dialect=uri.split(":")[0] if uri and ":" in uri else None,

386 folder=str(folder) if folder is not None else None,

387 migrate=migrate,

388 fake_migrate=fake_migrate,

389 caching=enable_typedal_caching,

390 pool_size=pool_size,

391 )

392

393 self._config = config

394

395 if config.folder:

396 Path(config.folder).mkdir(exist_ok=True)

397

398 super().__init__(

399 config.database,

400 config.pool_size,

401 config.folder,

402 db_codec,

403 check_reserved,

404 config.migrate,

405 config.fake_migrate,

406 migrate_enabled,

407 fake_migrate_all,

408 decode_credentials,

409 driver_args,

410 adapter_args,

411 attempts,

412 auto_import,

413 bigint_id,

414 debug,

415 lazy_tables,

416 db_uid,

417 after_connection,

418 tables,

419 ignore_field_case,

420 entity_quoting,

421 table_hash,

422 )

423

424 if config.caching:

425 self.try_define(_TypedalCache)

426 self.try_define(_TypedalCacheDependency)

427

428 def try_define(self, model: typing.Type[T], verbose: bool = False) -> typing.Type[T]:

429 """

430 Try to define a model with migrate or fall back to fake migrate.

431 """

432 try:

433 return self.define(model, migrate=True)

434 except Exception as e:

435 # clean up:

436 self.rollback()

437 if (tablename := self.to_snake(model.__name__)) and tablename in dir(self):

438 delattr(self, tablename)

439

440 if verbose:

441 warnings.warn(f"{model} could not be migrated, try faking", source=e, category=RuntimeWarning)

442

443 # try again:

444 return self.define(model, migrate=True, fake_migrate=True, redefine=True)

445

446 default_kwargs: typing.ClassVar[AnyDict] = {

447 # fields are 'required' (notnull) by default:

448 "notnull": True,

449 }

450

451 # maps table name to typedal class, for resolving future references

452 _class_map: typing.ClassVar[dict[str, typing.Type["TypedTable"]]] = {}

453

454 def _define(self, cls: typing.Type[T], **kwargs: Any) -> typing.Type[T]:

455 # todo: new relationship item added should also invalidate (previously unrelated) cache result

456

457 # todo: option to enable/disable cache dependency behavior:

458 # - don't set _before_update and _before_delete

459 # - don't add TypedalCacheDependency entry

460 # - don't invalidate other item on new row of this type

461

462 # when __future__.annotations is implemented, cls.__annotations__ will not work anymore as below.

463 # proper way to handle this would be (but gives error right now due to Table implementing magic methods):

464 # typing.get_type_hints(cls, globalns=None, localns=None)

465

466 # dirty way (with evil eval):

467 # [eval(v) for k, v in cls.__annotations__.items()]

468 # this however also stops working when variables outside this scope or even references to other

469 # objects are used. So for now, this package will NOT work when from __future__ import annotations is used,

470 # and might break in the future, when this annotations behavior is enabled by default.

471

472 # non-annotated variables have to be passed to define_table as kwargs

473 full_dict = all_dict(cls) # includes properties from parents (e.g. useful for mixins)

474

475 tablename = self.to_snake(cls.__name__)

476 # grab annotations of cls and it's parents:

477 annotations = all_annotations(cls)

478 # extend with `prop = TypedField()` 'annotations':

479 annotations |= {k: typing.cast(type, v) for k, v in full_dict.items() if is_typed_field(v)}

480 # remove internal stuff:

481 annotations = {k: v for k, v in annotations.items() if not k.startswith("_")}

482

483 typedfields: dict[str, TypedField[Any]] = {

484 k: instanciate(v, True) for k, v in annotations.items() if is_typed_field(v)

485 }

486

487 relationships: dict[str, type[Relationship[Any]]] = filter_out(annotations, Relationship)

488

489 fields = {fname: self._to_field(fname, ftype) for fname, ftype in annotations.items()}

490

491 # ! dont' use full_dict here:

492 other_kwargs = kwargs | {

493 k: v for k, v in cls.__dict__.items() if k not in annotations and not k.startswith("_")

494 } # other_kwargs was previously used to pass kwargs to typedal, but use @define(**kwargs) for that.

495 # now it's only used to extract relationships from the object.

496 # other properties of the class (incl methods) should not be touched

497

498 # for key in typedfields.keys() - full_dict.keys():

499 # # typed fields that don't haven't been added to the object yet

500 # setattr(cls, key, typedfields[key])

501

502 for key, field in typedfields.items():

503 # clone every property so it can be re-used across mixins:

504 clone = copy(field)

505 setattr(cls, key, clone)

506 typedfields[key] = clone

507

508 # start with base classes and overwrite with current class:

509 relationships = filter_out(full_dict, Relationship) | relationships | filter_out(other_kwargs, Relationship)

510

511 # DEPRECATED: Relationship as annotation is currently not supported!

512 # ensure they are all instances and

513 # not mix of instances (`= relationship()`) and classes (`: Relationship[...]`):

514 # relationships = {

515 # k: v if isinstance(v, Relationship) else to_relationship(cls, k, v) for k, v in relationships.items()

516 # }

517

518 # keys of implicit references (also relationships):

519 reference_field_keys = [k for k, v in fields.items() if v.type.split(" ")[0] in ("list:reference", "reference")]

520

521 # add implicit relationships:

522 # User; list[User]; TypedField[User]; TypedField[list[User]]

523 relationships |= {

524 k: new_relationship

525 for k in reference_field_keys

526 if k not in relationships and (new_relationship := to_relationship(cls, k, annotations[k]))

527 }

528

529 cache_dependency = self._config.caching and kwargs.pop("cache_dependency", True)

530

531 table: Table = self.define_table(tablename, *fields.values(), **kwargs)

532

533 for name, typed_field in typedfields.items():

534 field = fields[name]

535 typed_field.bind(field, table)

536

537 if issubclass(cls, TypedTable):

538 cls.__set_internals__(

539 db=self,

540 table=table,

541 # by now, all relationships should be instances!

542 relationships=typing.cast(dict[str, Relationship[Any]], relationships),

543 )

544 self._class_map[str(table)] = cls

545 cls.__on_define__(self)

546 else:

547 warnings.warn("db.define used without inheriting TypedTable. This could lead to strange problems!")

548

549 if not tablename.startswith("typedal_") and cache_dependency:

550 table._before_update.append(lambda s, _: _remove_cache(s, tablename))

551 table._before_delete.append(lambda s: _remove_cache(s, tablename))

552

553 return cls

554

555 @typing.overload

556 def define(self, maybe_cls: None = None, **kwargs: Any) -> typing.Callable[[typing.Type[T]], typing.Type[T]]:

557 """

558 Typing Overload for define without a class.

559

560 @db.define()

561 class MyTable(TypedTable): ...

562 """

563

564 @typing.overload

565 def define(self, maybe_cls: typing.Type[T], **kwargs: Any) -> typing.Type[T]:

566 """

567 Typing Overload for define with a class.

568

569 @db.define

570 class MyTable(TypedTable): ...

571 """

572

573 def define(

574 self, maybe_cls: typing.Type[T] | None = None, **kwargs: Any

575 ) -> typing.Type[T] | typing.Callable[[typing.Type[T]], typing.Type[T]]:

576 """

577 Can be used as a decorator on a class that inherits `TypedTable`, \

578 or as a regular method if you need to define your classes before you have access to a 'db' instance.

579

580 You can also pass extra arguments to db.define_table.

581 See http://www.web2py.com/books/default/chapter/29/06/the-database-abstraction-layer#Table-constructor

582

583 Example:

584 @db.define

585 class Person(TypedTable):

586 ...

587

588 class Article(TypedTable):

589 ...

590

591 # at a later time:

592 db.define(Article)

593

594 Returns:

595 the result of pydal.define_table

596 """

597

598 def wrapper(cls: typing.Type[T]) -> typing.Type[T]:

599 return self._define(cls, **kwargs)

600

601 if maybe_cls:

602 return wrapper(maybe_cls)

603

604 return wrapper

605

606 # def drop(self, table_name: str) -> None:

607 # """

608 # Remove a table by name (both on the database level and the typedal level).

609 # """

610 # # drop calls TypedTable.drop() and removes it from the `_class_map`

611 # if cls := self._class_map.pop(table_name, None):

612 # cls.drop()

613

614 # def drop_all(self, max_retries: int = None) -> None:

615 # """

616 # Remove all tables and keep doing so until everything is gone!

617 # """

618 # retries = 0

619 # if max_retries is None:

620 # max_retries = len(self.tables)

621 #

622 # while self.tables:

623 # retries += 1

624 # for table in self.tables:

625 # self.drop(table)

626 #

627 # if retries > max_retries:

628 # raise RuntimeError("Could not delete all tables")

629

630 def __call__(self, *_args: T_Query, **kwargs: Any) -> "TypedSet":

631 """

632 A db instance can be called directly to perform a query.

633

634 Usually, only a query is passed.

635

636 Example:

637 db(query).select()

638

639 """

640 args = list(_args)

641 if args:

642 cls = args[0]

643 if isinstance(cls, bool):

644 raise ValueError("Don't actually pass a bool to db()! Use a query instead.")

645

646 if isinstance(cls, type) and issubclass(type(cls), type) and issubclass(cls, TypedTable):

647 # table defined without @db.define decorator!

648 _cls: typing.Type[TypedTable] = cls

649 args[0] = _cls.id != None

650

651 _set = super().__call__(*args, **kwargs)

652 return typing.cast(TypedSet, _set)

653

654 def __getitem__(self, key: str) -> "Table":

655 """

656 Allows dynamically accessing a table by its name as a string.

657

658 Example:

659 db['users'] -> user

660 """

661 return typing.cast(Table, super().__getitem__(str(key)))

662

663 @classmethod

664 def _build_field(cls, name: str, _type: str, **kw: Any) -> Field:

665 return Field(name, _type, **{**cls.default_kwargs, **kw})

666

667 @classmethod

668 def _annotation_to_pydal_fieldtype(

669 cls, _ftype: T_annotation, mut_kw: typing.MutableMapping[str, Any]

670 ) -> Optional[str]:

671 # ftype can be a union or type. typing.cast is sometimes used to tell mypy when it's not a union.

672 ftype = typing.cast(type, _ftype) # cast from typing.Type to type to make mypy happy)

673

674 if isinstance(ftype, str):

675 # extract type from string

676 ftype = typing.get_args(typing.Type[ftype])[0]._evaluate(

677 localns=locals(), globalns=globals(), recursive_guard=frozenset()

678 )

679

680 if mapping := BASIC_MAPPINGS.get(ftype):

681 # basi types

682 return mapping

683 elif isinstance(ftype, _Table):

684 # db.table

685 return f"reference {ftype._tablename}"

686 elif issubclass(type(ftype), type) and issubclass(ftype, TypedTable):

687 # SomeTable

688 snakename = cls.to_snake(ftype.__name__)

689 return f"reference {snakename}"

690 elif isinstance(ftype, TypedField):

691 # FieldType(type, ...)

692 return ftype._to_field(mut_kw)

693 elif origin_is_subclass(ftype, TypedField):

694 # TypedField[int]

695 return cls._annotation_to_pydal_fieldtype(typing.get_args(ftype)[0], mut_kw)

696 elif isinstance(ftype, types.GenericAlias) and typing.get_origin(ftype) in (list, TypedField):

697 # list[str] -> str -> string -> list:string

698 _child_type = typing.get_args(ftype)[0]

699 _child_type = cls._annotation_to_pydal_fieldtype(_child_type, mut_kw)

700 return f"list:{_child_type}"

701 elif is_union(ftype):

702 # str | int -> UnionType

703 # typing.Union[str | int] -> typing._UnionGenericAlias

704

705 # Optional[type] == type | None

706

707 match typing.get_args(ftype):

708 case (_child_type, _Types.NONETYPE) | (_Types.NONETYPE, _child_type):

709 # good union of Nullable

710

711 # if a field is optional, it is nullable:

712 mut_kw["notnull"] = False

713 return cls._annotation_to_pydal_fieldtype(_child_type, mut_kw)

714 case _:

715 # two types is not supported by the db!

716 return None

717 else:

718 return None

719

720 @classmethod

721 def _to_field(cls, fname: str, ftype: type, **kw: Any) -> Field:

722 """

723 Convert a annotation into a pydal Field.

724

725 Args:

726 fname: name of the property

727 ftype: annotation of the property

728 kw: when using TypedField or a function returning it (e.g. StringField),

729 keyword args can be used to pass any other settings you would normally to a pydal Field

730

731 -> pydal.Field(fname, ftype, **kw)

732

733 Example:

734 class MyTable:

735 fname: ftype

736 id: int

737 name: str

738 reference: Table

739 other: TypedField(str, default="John Doe") # default will be in kwargs

740 """

741 fname = cls.to_snake(fname)

742

743 if converted_type := cls._annotation_to_pydal_fieldtype(ftype, kw):

744 return cls._build_field(fname, converted_type, **kw)

745 else:

746 raise NotImplementedError(f"Unsupported type {ftype}/{type(ftype)}")

747

748 @staticmethod

749 def to_snake(camel: str) -> str:

750 """

751 Moved to helpers, kept as a static method for legacy reasons.

752 """

753 return to_snake(camel)

754

755

756 class TableProtocol(typing.Protocol): # pragma: no cover

757 """

758 Make mypy happy.

759 """

760

761 id: "TypedField[int]"

762

763 def __getitem__(self, item: str) -> Field:

764 """

765 Tell mypy a Table supports dictionary notation for columns.

766 """

767

768

769 class Table(_Table, TableProtocol): # type: ignore

770 """

771 Make mypy happy.

772 """

773

774

775 class TableMeta(type):

776 """

777 This metaclass contains functionality on table classes, that doesn't exist on its instances.

778

779 Example:

780 class MyTable(TypedTable):

781 some_field: TypedField[int]

782

783 MyTable.update_or_insert(...) # should work

784

785 MyTable.some_field # -> Field, can be used to query etc.

786

787 row = MyTable.first() # returns instance of MyTable

788

789 # row.update_or_insert(...) # shouldn't work!

790

791 row.some_field # -> int, with actual data

792

793 """

794

795 # set up by db.define:

796 # _db: TypeDAL | None = None

797 # _table: Table | None = None

798 _db: TypeDAL | None = None

799 _table: Table | None = None

800 _relationships: dict[str, Relationship[Any]] | None = None

801

802 #########################

803 # TypeDAL custom logic: #

804 #########################

805

806 def __set_internals__(self, db: pydal.DAL, table: Table, relationships: dict[str, Relationship[Any]]) -> None:

807 """

808 Store the related database and pydal table for later usage.

809 """

810 self._db = db

811 self._table = table

812 self._relationships = relationships

813

814 def __getattr__(self, col: str) -> Optional[Field]:

815 """

816 Magic method used by TypedTableMeta to get a database field with dot notation on a class.

817

818 Example:

819 SomeTypedTable.col -> db.table.col (via TypedTableMeta.__getattr__)

820

821 """

822 if self._table:

823 return getattr(self._table, col, None)

824

825 return None

826

827 def _ensure_table_defined(self) -> Table:

828 if not self._table:

829 raise EnvironmentError("@define or db.define is not called on this class yet!")

830 return self._table

831

832 def __iter__(self) -> typing.Generator[Field, None, None]:

833 """

834 Loop through the columns of this model.

835 """

836 table = self._ensure_table_defined()

837 yield from iter(table)

838

839 def __getitem__(self, item: str) -> Field:

840 """

841 Allow dict notation to get a column of this table (-> Field instance).

842 """

843 table = self._ensure_table_defined()

844 return table[item]

845

846 def __str__(self) -> str:

847 """

848 Normally, just returns the underlying table name, but with a fallback if the model is unbound.

849 """

850 if self._table:

851 return str(self._table)

852 else:

853 return f"<unbound table {self.__name__}>"

854

855 def from_row(self: typing.Type[T_MetaInstance], row: pydal.objects.Row) -> T_MetaInstance:

856 """

857 Create a model instance from a pydal row.

858 """

859 return self(row)

860

861 def all(self: typing.Type[T_MetaInstance]) -> "TypedRows[T_MetaInstance]":

862 """

863 Return all rows for this model.

864 """

865 return self.collect()

866

867 def get_relationships(self) -> dict[str, Relationship[Any]]:

868 """

869 Return the registered relationships of the current model.

870 """

871 return self._relationships or {}

872

873 ##########################

874 # TypeDAL Modified Logic #

875 ##########################

876

877 def insert(self: typing.Type[T_MetaInstance], **fields: Any) -> T_MetaInstance:

878 """

879 This is only called when db.define is not used as a decorator.

880

881 cls.__table functions as 'self'

882

883 Args:

884 **fields: anything you want to insert in the database

885

886 Returns: the ID of the new row.

887

888 """

889 table = self._ensure_table_defined()

890

891 result = table.insert(**fields)

892 # it already is an int but mypy doesn't understand that

893 return self(result)

894

895 def _insert(self, **fields: Any) -> str:

896 table = self._ensure_table_defined()

897

898 return str(table._insert(**fields))

899

900 def bulk_insert(self: typing.Type[T_MetaInstance], items: list[AnyDict]) -> "TypedRows[T_MetaInstance]":

901 """

902 Insert multiple rows, returns a TypedRows set of new instances.

903 """

904 table = self._ensure_table_defined()

905 result = table.bulk_insert(items)

906 return self.where(lambda row: row.id.belongs(result)).collect()

907

908 def update_or_insert(

909 self: typing.Type[T_MetaInstance], query: T_Query | AnyDict = DEFAULT, **values: Any

910 ) -> T_MetaInstance:

911 """

912 Update a row if query matches, else insert a new one.

913

914 Returns the created or updated instance.

915 """

916 table = self._ensure_table_defined()

917

918 if query is DEFAULT:

919 record = table(**values)

920 elif isinstance(query, dict):

921 record = table(**query)

922 else:

923 record = table(query)

924

925 if not record:

926 return self.insert(**values)

927

928 record.update_record(**values)

929 return self(record)

930

931 def validate_and_insert(

932 self: typing.Type[T_MetaInstance], **fields: Any

933 ) -> tuple[Optional[T_MetaInstance], Optional[dict[str, str]]]:

934 """

935 Validate input data and then insert a row.

936

937 Returns a tuple of (the created instance, a dict of errors).

938 """

939 table = self._ensure_table_defined()

940 result = table.validate_and_insert(**fields)

941 if row_id := result.get("id"):

942 return self(row_id), None

943 else:

944 return None, result.get("errors")

945

946 def validate_and_update(

947 self: typing.Type[T_MetaInstance], query: Query, **fields: Any

948 ) -> tuple[Optional[T_MetaInstance], Optional[dict[str, str]]]:

949 """

950 Validate input data and then update max 1 row.

951

952 Returns a tuple of (the updated instance, a dict of errors).

953 """

954 table = self._ensure_table_defined()

955

956 result = table.validate_and_update(query, **fields)

957

958 if errors := result.get("errors"):

959 return None, errors

960 elif row_id := result.get("id"):

961 return self(row_id), None

962 else: # pragma: no cover

963 # update on query without result (shouldnt happen)

964 return None, None

965

966 def validate_and_update_or_insert(

967 self: typing.Type[T_MetaInstance], query: Query, **fields: Any

968 ) -> tuple[Optional[T_MetaInstance], Optional[dict[str, str]]]:

969 """

970 Validate input data and then update_and_insert (on max 1 row).

971

972 Returns a tuple of (the updated/created instance, a dict of errors).

973 """

974 table = self._ensure_table_defined()

975 result = table.validate_and_update_or_insert(query, **fields)

976

977 if errors := result.get("errors"):

978 return None, errors

979 elif row_id := result.get("id"):

980 return self(row_id), None

981 else: # pragma: no cover

982 # update on query without result (shouldnt happen)

983 return None, None

984

985 def select(self: typing.Type[T_MetaInstance], *a: Any, **kw: Any) -> "QueryBuilder[T_MetaInstance]":

986 """

987 See QueryBuilder.select!

988 """

989 return QueryBuilder(self).select(*a, **kw)

990

991 def paginate(self: typing.Type[T_MetaInstance], limit: int, page: int = 1) -> "PaginatedRows[T_MetaInstance]":

992 """

993 See QueryBuilder.paginate!

994 """

995 return QueryBuilder(self).paginate(limit=limit, page=page)

996

997 def chunk(

998 self: typing.Type[T_MetaInstance], chunk_size: int

999 ) -> typing.Generator["TypedRows[T_MetaInstance]", Any, None]:

1000 """

1001 See QueryBuilder.chunk!

1002 """

1003 return QueryBuilder(self).chunk(chunk_size)

1004

1005 def where(self: typing.Type[T_MetaInstance], *a: Any, **kw: Any) -> "QueryBuilder[T_MetaInstance]":

1006 """

1007 See QueryBuilder.where!

1008 """

1009 return QueryBuilder(self).where(*a, **kw)

1010

1011 def cache(self: typing.Type[T_MetaInstance], *deps: Any, **kwargs: Any) -> "QueryBuilder[T_MetaInstance]":

1012 """

1013 See QueryBuilder.cache!

1014 """

1015 return QueryBuilder(self).cache(*deps, **kwargs)

1016

1017 def count(self: typing.Type[T_MetaInstance]) -> int:

1018 """

1019 See QueryBuilder.count!

1020 """

1021 return QueryBuilder(self).count()

1022

1023 def first(self: typing.Type[T_MetaInstance]) -> T_MetaInstance | None:

1024 """

1025 See QueryBuilder.first!

1026 """

1027 return QueryBuilder(self).first()

1028

1029 def join(

1030 self: typing.Type[T_MetaInstance],

1031 *fields: str | typing.Type["TypedTable"],

1032 method: JOIN_OPTIONS = None,

1033 on: OnQuery | list[Expression] | Expression = None,

1034 condition: Condition = None,

1035 ) -> "QueryBuilder[T_MetaInstance]":

1036 """

1037 See QueryBuilder.join!

1038 """

1039 return QueryBuilder(self).join(*fields, on=on, condition=condition, method=method)

1040

1041 def collect(self: typing.Type[T_MetaInstance], verbose: bool = False) -> "TypedRows[T_MetaInstance]":

1042 """

1043 See QueryBuilder.collect!

1044 """

1045 return QueryBuilder(self).collect(verbose=verbose)

1046

1047 @property

1048 def ALL(cls) -> pydal.objects.SQLALL:

1049 """

1050 Select all fields for this table.

1051 """

1052 table = cls._ensure_table_defined()

1053

1054 return table.ALL

1055

1056 ##########################

1057 # TypeDAL Shadowed Logic #

1058 ##########################

1059 fields: list[str]

1060

1061 # other table methods:

1062

1063 def truncate(self, mode: str = "") -> None:

1064 """

1065 Remove all data and reset index.

1066 """

1067 table = self._ensure_table_defined()

1068 table.truncate(mode)

1069

1070 def drop(self, mode: str = "") -> None:

1071 """

1072 Remove the underlying table.

1073 """

1074 table = self._ensure_table_defined()

1075 table.drop(mode)

1076

1077 def create_index(self, name: str, *fields: Field | str, **kwargs: Any) -> bool:

1078 """

1079 Add an index on some columns of this table.

1080 """

1081 table = self._ensure_table_defined()

1082 result = table.create_index(name, *fields, **kwargs)

1083 return typing.cast(bool, result)

1084

1085 def drop_index(self, name: str, if_exists: bool = False) -> bool:

1086 """

1087 Remove an index from this table.

1088 """

1089 table = self._ensure_table_defined()

1090 result = table.drop_index(name, if_exists)

1091 return typing.cast(bool, result)

1092

1093 def import_from_csv_file(

1094 self,

1095 csvfile: typing.TextIO,

1096 id_map: dict[str, str] = None,

1097 null: Any = "<NULL>",

1098 unique: str = "uuid",

1099 id_offset: dict[str, int] = None, # id_offset used only when id_map is None

1100 transform: typing.Callable[[dict[Any, Any]], dict[Any, Any]] = None,

1101 validate: bool = False,

1102 encoding: str = "utf-8",

1103 delimiter: str = ",",

1104 quotechar: str = '"',

1105 quoting: int = csv.QUOTE_MINIMAL,

1106 restore: bool = False,

1107 **kwargs: Any,

1108 ) -> None:

1109 """

1110 Load a csv file into the database.

1111 """

1112 table = self._ensure_table_defined()

1113 table.import_from_csv_file(

1114 csvfile,

1115 id_map=id_map,

1116 null=null,

1117 unique=unique,

1118 id_offset=id_offset,

1119 transform=transform,

1120 validate=validate,

1121 encoding=encoding,

1122 delimiter=delimiter,

1123 quotechar=quotechar,

1124 quoting=quoting,

1125 restore=restore,

1126 **kwargs,

1127 )

1128

1129 def on(self, query: Query | bool) -> Expression:

1130 """

1131 Shadow Table.on.

1132

1133 Used for joins.

1134

1135 See Also:

1136 http://web2py.com/books/default/chapter/29/06/the-database-abstraction-layer?search=export_to_csv_file#One-to-many-relation

1137 """

1138 table = self._ensure_table_defined()

1139 return typing.cast(Expression, table.on(query))

1140

1141 def with_alias(self, alias: str) -> _Table:

1142 """

1143 Shadow Table.with_alias.

1144

1145 Useful for joins when joining the same table multiple times.

1146

1147 See Also:

1148 http://web2py.com/books/default/chapter/29/06/the-database-abstraction-layer?search=export_to_csv_file#One-to-many-relation

1149 """

1150 table = self._ensure_table_defined()

1151 return table.with_alias(alias)

1152

1153 # @typing.dataclass_transform()

1154

1155

1156 class TypedField(typing.Generic[T_Value]): # pragma: no cover

1157 """

1158 Typed version of pydal.Field, which will be converted to a normal Field in the background.

1159 """

1160

1161 # will be set by .bind on db.define

1162 name = ""

1163 _db: Optional[pydal.DAL] = None

1164 _rname: Optional[str] = None

1165 _table: Optional[Table] = None

1166 _field: Optional[Field] = None

1167

1168 _type: T_annotation

1169 kwargs: Any

1170

1171 requires: Validator | typing.Iterable[Validator]

1172

1173 def __init__(self, _type: typing.Type[T_Value] | types.UnionType = str, /, **settings: Any) -> None: # type: ignore

1174 """

1175 A TypedFieldType should not be inited manually, but TypedField (from `fields.py`) should be used!

1176 """

1177 self._type = _type

1178 self.kwargs = settings

1179 super().__init__()

1180

1181 @typing.overload

1182 def __get__(self, instance: T_MetaInstance, owner: typing.Type[T_MetaInstance]) -> T_Value: # pragma: no cover

1183 """

1184 row.field -> (actual data).

1185 """

1186

1187 @typing.overload

1188 def __get__(self, instance: None, owner: "typing.Type[TypedTable]") -> "TypedField[T_Value]": # pragma: no cover

1189 """

1190 Table.field -> Field.

1191 """

1192

1193 def __get__(

1194 self, instance: T_MetaInstance | None, owner: typing.Type[T_MetaInstance]

1195 ) -> typing.Union[T_Value, "TypedField[T_Value]"]:

1196 """

1197 Since this class is a Descriptor field, \

1198 it returns something else depending on if it's called on a class or instance.

1199

1200 (this is mostly for mypy/typing)

1201 """

1202 if instance:

1203 # this is only reached in a very specific case:

1204 # an instance of the object was created with a specific set of fields selected (excluding the current one)

1205 # in that case, no value was stored in the owner -> return None (since the field was not selected)

1206 return typing.cast(T_Value, None) # cast as T_Value so mypy understands it for selected fields

1207 else:

1208 # getting as class -> return actual field so pydal understands it when using in query etc.

1209 return typing.cast(TypedField[T_Value], self._field) # pretend it's still typed for IDE support

1210

1211 def __str__(self) -> str:

1212 """

1213 String representation of a Typed Field.

1214

1215 If `type` is set explicitly (e.g. TypedField(str, type="text")), that type is used: `TypedField.text`,

1216 otherwise the type annotation is used (e.g. TypedField(str) -> TypedField.str)

1217 """

1218 return str(self._field) if self._field else ""

1219

1220 def __repr__(self) -> str:

1221 """

1222 More detailed string representation of a Typed Field.

1223

1224 Uses __str__ and adds the provided extra options (kwargs) in the representation.

1225 """

1226 s = self.__str__()

1227

1228 if "type" in self.kwargs:

1229 # manual type in kwargs supplied

1230 t = self.kwargs["type"]

1231 elif issubclass(type, type(self._type)):

1232 # normal type, str.__name__ = 'str'

1233 t = getattr(self._type, "__name__", str(self._type))

1234 elif t_args := typing.get_args(self._type):

1235 # list[str] -> 'str'

1236 t = t_args[0].__name__

1237 else: # pragma: no cover

1238 # fallback - something else, may not even happen, I'm not sure

1239 t = self._type

1240

1241 s = f"TypedField[{t}].{s}" if s else f"TypedField[{t}]"

1242

1243 kw = self.kwargs.copy()

1244 kw.pop("type", None)

1245 return f"<{s} with options {kw}>"

1246

1247 def _to_field(self, extra_kwargs: typing.MutableMapping[str, Any]) -> Optional[str]:

1248 """

1249 Convert a Typed Field instance to a pydal.Field.

1250 """

1251 other_kwargs = self.kwargs.copy()

1252 extra_kwargs.update(other_kwargs)

1253 return extra_kwargs.pop("type", False) or TypeDAL._annotation_to_pydal_fieldtype(self._type, extra_kwargs)

1254

1255 def bind(self, field: pydal.objects.Field, table: pydal.objects.Table) -> None:

1256 """

1257 Bind the right db/table/field info to this class, so queries can be made using `Class.field == ...`.

1258 """

1259 self._table = table

1260 self._field = field

1261

1262 def __getattr__(self, key: str) -> Any:

1263 """

1264 If the regular getattribute does not work, try to get info from the related Field.

1265 """

1266 with contextlib.suppress(AttributeError):

1267 return super().__getattribute__(key)

1268

1269 # try on actual field:

1270 return getattr(self._field, key)

1271

1272 def __eq__(self, other: Any) -> Query:

1273 """

1274 Performing == on a Field will result in a Query.

1275 """

1276 return typing.cast(Query, self._field == other)

1277

1278 def __ne__(self, other: Any) -> Query:

1279 """

1280 Performing != on a Field will result in a Query.

1281 """

1282 return typing.cast(Query, self._field != other)

1283

1284 def __gt__(self, other: Any) -> Query:

1285 """

1286 Performing > on a Field will result in a Query.

1287 """

1288 return typing.cast(Query, self._field > other)

1289

1290 def __lt__(self, other: Any) -> Query:

1291 """

1292 Performing < on a Field will result in a Query.

1293 """

1294 return typing.cast(Query, self._field < other)

1295

1296 def __ge__(self, other: Any) -> Query:

1297 """

1298 Performing >= on a Field will result in a Query.

1299 """

1300 return typing.cast(Query, self._field >= other)

1301

1302 def __le__(self, other: Any) -> Query:

1303 """

1304 Performing <= on a Field will result in a Query.

1305 """

1306 return typing.cast(Query, self._field <= other)

1307

1308 def __hash__(self) -> int:

1309 """

1310 Shadow Field.__hash__.

1311 """

1312 return hash(self._field)

1313

1314 def __invert__(self) -> Expression:

1315 """

1316 Performing ~ on a Field will result in an Expression.

1317 """

1318 if not self._field: # pragma: no cover

1319 raise ValueError("Unbound Field can not be inverted!")

1320

1321 return typing.cast(Expression, ~self._field)

1322

1323

1324 class _TypedTable:

1325 """

1326 This class is a final shared parent between TypedTable and Mixins.

1327

1328 This needs to exist because otherwise the __on_define__ of Mixins are not executed.

1329 Notably, this class exists at a level ABOVE the `metaclass=TableMeta`,

1330 because otherwise typing gets confused when Mixins are used and multiple types could satisfy

1331 generic 'T subclass of TypedTable'

1332 -> Setting 'TypedTable' as the parent for Mixin does not work at runtime (and works semi at type check time)

1333 """

1334

1335 id: "TypedField[int]"

1336

1337 _before_insert: list[BeforeInsertCallable]

1338 _after_insert: list[AfterInsertCallable]

1339 _before_update: list[BeforeUpdateCallable]

1340 _after_update: list[AfterUpdateCallable]

1341 _before_delete: list[BeforeDeleteCallable]

1342 _after_delete: list[AfterDeleteCallable]

1343

1344 @classmethod

1345 def __on_define__(cls, db: TypeDAL) -> None:

1346 """

1347 Method that can be implemented by tables to do an action after db.define is completed.

1348

1349 This can be useful if you need to add something like requires=IS_NOT_IN_DB(db, "table.field"),

1350 where you need a reference to the current database, which may not exist yet when defining the model.

1351 """

1352

1353

1354 class TypedTable(_TypedTable, metaclass=TableMeta):

1355 """

1356 Enhanded modeling system on top of pydal's Table that adds typing and additional functionality.

1357 """

1358

1359 # set up by 'new':

1360 _row: Row | None = None

1361

1362 _with: list[str]

1363

1364 def _setup_instance_methods(self) -> None:

1365 self.as_dict = self._as_dict # type: ignore

1366 self.__json__ = self.as_json = self._as_json # type: ignore

1367 # self.as_yaml = self._as_yaml # type: ignore

1368 self.as_xml = self._as_xml # type: ignore

1369

1370 self.update = self._update # type: ignore

1371

1372 self.delete_record = self._delete_record # type: ignore

1373 self.update_record = self._update_record # type: ignore

1374

1375 def __new__(

1376 cls, row_or_id: typing.Union[Row, Query, pydal.objects.Set, int, str, None, "TypedTable"] = None, **filters: Any

1377 ) -> "TypedTable":

1378 """

1379 Create a Typed Rows model instance from an existing row, ID or query.

1380

1381 Examples:

1382 MyTable(1)

1383 MyTable(id=1)

1384 MyTable(MyTable.id == 1)

1385 """

1386 table = cls._ensure_table_defined()

1387 inst = super().__new__(cls)

1388

1389 if isinstance(row_or_id, TypedTable):

1390 # existing typed table instance!

1391 return row_or_id

1392 elif isinstance(row_or_id, pydal.objects.Row):

1393 row = row_or_id

1394 elif row_or_id is not None:

1395 row = table(row_or_id, **filters)

1396 elif filters:

1397 row = table(**filters)

1398 else:

1399 # dummy object

1400 return inst

1401

1402 if not row:

1403 return None # type: ignore

1404

1405 inst._row = row

1406 inst.__dict__.update(row)

1407 inst._setup_instance_methods()

1408 return inst

1409

1410 def __iter__(self) -> typing.Generator[Any, None, None]:

1411 """

1412 Allows looping through the columns.

1413 """

1414 row = self._ensure_matching_row()

1415 yield from iter(row)

1416

1417 def __getitem__(self, item: str) -> Any:

1418 """

1419 Allows dictionary notation to get columns.

1420 """

1421 if item in self.__dict__:

1422 return self.__dict__.get(item)

1423

1424 # fallback to lookup in row

1425 if self._row:

1426 return self._row[item]

1427

1428 # nothing found!

1429 raise KeyError(item)

1430

1431 def __getattr__(self, item: str) -> Any:

1432 """

1433 Allows dot notation to get columns.

1434 """

1435 if value := self.get(item):

1436 return value

1437

1438 raise AttributeError(item)

1439

1440 def get(self, item: str, default: Any = None) -> Any:

1441 """

1442 Try to get a column from this instance, else return default.

1443 """

1444 try:

1445 return self.__getitem__(item)

1446 except KeyError:

1447 return default

1448

1449 def __setitem__(self, key: str, value: Any) -> None:

1450 """

1451 Data can both be updated via dot and dict notation.

1452 """

1453 return setattr(self, key, value)

1454

1455 def __int__(self) -> int:

1456 """

1457 Calling int on a model instance will return its id.

1458 """

1459 return getattr(self, "id", 0)

1460

1461 def __bool__(self) -> bool:

1462 """

1463 If the instance has an underlying row with data, it is truthy.

1464 """

1465 return bool(getattr(self, "_row", False))

1466

1467 def _ensure_matching_row(self) -> Row:

1468 if not getattr(self, "_row", None):

1469 raise EnvironmentError("Trying to access non-existant row. Maybe it was deleted or not yet initialized?")

1470 return self._row

1471

1472 def __repr__(self) -> str:

1473 """

1474 String representation of the model instance.

1475 """

1476 model_name = self.__class__.__name__

1477 model_data = {}

1478

1479 if self._row:

1480 model_data = self._row.as_json()

1481

1482 details = model_name

1483 details += f"({model_data})"

1484

1485 if relationships := getattr(self, "_with", []):

1486 details += f" + {relationships}"

1487

1488 return f"<{details}>"

1489

1490 # serialization

1491 # underscore variants work for class instances (set up by _setup_instance_methods)

1492

1493 @classmethod

1494 def as_dict(cls, flat: bool = False, sanitize: bool = True) -> AnyDict:

1495 """

1496 Dump the object to a plain dict.

1497

1498 Can be used as both a class or instance method:

1499 - dumps the table info if it's a class

1500 - dumps the row info if it's an instance (see _as_dict)

1501 """

1502 table = cls._ensure_table_defined()

1503 result = table.as_dict(flat, sanitize)

1504 return typing.cast(AnyDict, result)

1505

1506 @classmethod

1507 def as_json(cls, sanitize: bool = True, indent: Optional[int] = None, **kwargs: Any) -> str:

1508 """

1509 Dump the object to json.

1510

1511 Can be used as both a class or instance method:

1512 - dumps the table info if it's a class

1513 - dumps the row info if it's an instance (see _as_json)

1514 """

1515 data = cls.as_dict(sanitize=sanitize)

1516 return as_json.encode(data, indent=indent, **kwargs)

1517

1518 @classmethod

1519 def as_xml(cls, sanitize: bool = True) -> str: # pragma: no cover

1520 """

1521 Dump the object to xml.

1522

1523 Can be used as both a class or instance method:

1524 - dumps the table info if it's a class

1525 - dumps the row info if it's an instance (see _as_xml)

1526 """

1527 table = cls._ensure_table_defined()

1528 return typing.cast(str, table.as_xml(sanitize))

1529

1530 @classmethod

1531 def as_yaml(cls, sanitize: bool = True) -> str:

1532 """

1533 Dump the object to yaml.

1534

1535 Can be used as both a class or instance method:

1536 - dumps the table info if it's a class

1537 - dumps the row info if it's an instance (see _as_yaml)

1538 """

1539 table = cls._ensure_table_defined()

1540 return typing.cast(str, table.as_yaml(sanitize))

1541

1542 def _as_dict(

1543 self, datetime_to_str: bool = False, custom_types: typing.Iterable[type] | type | None = None

1544 ) -> AnyDict:

1545 row = self._ensure_matching_row()

1546

1547 result = row.as_dict(datetime_to_str=datetime_to_str, custom_types=custom_types)

1548

1549 def asdict_method(obj: Any) -> Any: # pragma: no cover

1550 if hasattr(obj, "_as_dict"): # typedal

1551 return obj._as_dict()

1552 elif hasattr(obj, "as_dict"): # pydal

1553 return obj.as_dict()

1554 else: # something else??

1555 return obj.__dict__

1556

1557 if _with := getattr(self, "_with", None):

1558 for relationship in _with:

1559 data = self.get(relationship)

1560

1561 if isinstance(data, list):

1562 data = [asdict_method(_) for _ in data]

1563 elif data:

1564 data = asdict_method(data)

1565

1566 result[relationship] = data

1567

1568 return typing.cast(AnyDict, result)

1569

1570 def _as_json(

1571 self,

1572 default: typing.Callable[[Any], Any] = None,

1573 indent: Optional[int] = None,

1574 **kwargs: Any,

1575 ) -> str:

1576 data = self._as_dict()

1577 return as_json.encode(data, default=default, indent=indent, **kwargs)

1578

1579 def _as_xml(self, sanitize: bool = True) -> str: # pragma: no cover

1580 row = self._ensure_matching_row()

1581 return typing.cast(str, row.as_xml(sanitize))

1582

1583 # def _as_yaml(self, sanitize: bool = True) -> str:

1584 # row = self._ensure_matching_row()

1585 # return typing.cast(str, row.as_yaml(sanitize))

1586

1587 def __setattr__(self, key: str, value: Any) -> None:

1588 """

1589 When setting a property on a Typed Table model instance, also update the underlying row.

1590 """

1591 if self._row and key in self._row.__dict__ and not callable(value):

1592 # enables `row.key = value; row.update_record()`

1593 self._row[key] = value

1594

1595 super().__setattr__(key, value)

1596

1597 @classmethod

1598 def update(cls: typing.Type[T_MetaInstance], query: Query, **fields: Any) -> T_MetaInstance | None:

1599 """

1600 Update one record.

1601

1602 Example:

1603 MyTable.update(MyTable.id == 1, name="NewName") -> MyTable

1604 """

1605 # todo: update multiple?

1606 if record := cls(query):

1607 return record.update_record(**fields)

1608 else:

1609 return None

1610

1611 def _update(self: T_MetaInstance, **fields: Any) -> T_MetaInstance:

1612 row = self._ensure_matching_row()

1613 row.update(**fields)

1614 self.__dict__.update(**fields)

1615 return self

1616

1617 def _update_record(self: T_MetaInstance, **fields: Any) -> T_MetaInstance:

1618 row = self._ensure_matching_row()

1619 new_row = row.update_record(**fields)

1620 self.update(**new_row)

1621 return self

1622

1623 def update_record(self: T_MetaInstance, **fields: Any) -> T_MetaInstance: # pragma: no cover

1624 """

1625 Here as a placeholder for _update_record.

1626

1627 Will be replaced on instance creation!

1628 """

1629 return self._update_record(**fields)

1630

1631 def _delete_record(self) -> int:

1632 """

1633 Actual logic in `pydal.helpers.classes.RecordDeleter`.

1634 """

1635 row = self._ensure_matching_row()

1636 result = row.delete_record()

1637 self.__dict__ = {} # empty self, since row is no more.

1638 self._row = None # just to be sure

1639 self._setup_instance_methods()

1640 # ^ instance methods might've been deleted by emptying dict,

1641 # but we still want .as_dict to show an error, not the table's as_dict.

1642 return typing.cast(int, result)

1643

1644 def delete_record(self) -> int: # pragma: no cover

1645 """

1646 Here as a placeholder for _delete_record.

1647

1648 Will be replaced on instance creation!

1649 """

1650 return self._delete_record()

1651

1652 # __del__ is also called on the end of a scope so don't remove records on every del!!

1653

1654 # pickling:

1655

1656 def __getstate__(self) -> AnyDict:

1657 """

1658 State to save when pickling.

1659

1660 Prevents db connection from being pickled.

1661 Similar to as_dict but without changing the data of the relationships (dill does that recursively)

1662 """

1663 row = self._ensure_matching_row()

1664 result: AnyDict = row.as_dict()

1665

1666 if _with := getattr(self, "_with", None):

1667 result["_with"] = _with

1668 for relationship in _with:

1669 data = self.get(relationship)

1670

1671 result[relationship] = data

1672

1673 result["_row"] = self._row.as_json() if self._row else ""

1674 return result

1675

1676 def __setstate__(self, state: AnyDict) -> None:

1677 """

1678 Used by dill when loading from a bytestring.

1679 """

1680 # as_dict also includes table info, so dump as json to only get the actual row data

1681 # then create a new (more empty) row object:

1682 state["_row"] = Row(json.loads(state["_row"]))

1683 self.__dict__ |= state

1684

1685

1686# backwards compat:

1687 TypedRow = TypedTable

1688

1689

1690 class TypedRows(typing.Collection[T_MetaInstance], Rows):

1691 """

1692 Slighly enhaned and typed functionality on top of pydal Rows (the result of a select).

1693 """

1694

1695 records: dict[int, T_MetaInstance]

1696 # _rows: Rows

1697 model: typing.Type[T_MetaInstance]

1698 metadata: Metadata

1699

1700 # pseudo-properties: actually stored in _rows

1701 db: TypeDAL

1702 colnames: list[str]

1703 fields: list[Field]

1704 colnames_fields: list[Field]

1705 response: list[tuple[Any, ...]]

1706

1707 def __init__(

1708 self,

1709 rows: Rows,

1710 model: typing.Type[T_MetaInstance],

1711 records: dict[int, T_MetaInstance] = None,

1712 metadata: Metadata = None,

1713 ) -> None:

1714 """

1715 Should not be called manually!

1716

1717 Normally, the `records` from an existing `Rows` object are used

1718 but these can be overwritten with a `records` dict.

1719 `metadata` can be any (un)structured data

1720 `model` is a Typed Table class

1721 """

1722 records = records or {row.id: model(row) for row in rows}

1723 super().__init__(rows.db, records, rows.colnames, rows.compact, rows.response, rows.fields)

1724 self.model = model

1725 self.metadata = metadata or {}

1726 self.colnames = rows.colnames

1727

1728 def __len__(self) -> int:

1729 """

1730 Return the count of rows.

1731 """

1732 return len(self.records)

1733

1734 def __iter__(self) -> typing.Iterator[T_MetaInstance]:

1735 """

1736 Loop through the rows.

1737 """

1738 yield from self.records.values()

1739

1740 def __contains__(self, ind: Any) -> bool:

1741 """

1742 Check if an id exists in this result set.

1743 """

1744 return ind in self.records

1745

1746 def first(self) -> T_MetaInstance | None:

1747 """

1748 Get the row with the lowest id.

1749 """

1750 if not self.records:

1751 return None

1752

1753 return next(iter(self))

1754

1755 def last(self) -> T_MetaInstance | None:

1756 """

1757 Get the row with the highest id.

1758 """

1759 if not self.records:

1760 return None

1761

1762 max_id = max(self.records.keys())

1763 return self[max_id]

1764

1765 def find(

1766 self, f: typing.Callable[[T_MetaInstance], Query], limitby: tuple[int, int] = None

1767 ) -> "TypedRows[T_MetaInstance]":

1768 """

1769 Returns a new Rows object, a subset of the original object, filtered by the function `f`.

1770 """

1771 if not self.records:

1772 return self.__class__(self, self.model, {})

1773

1774 records = {}

1775 if limitby:

1776 _min, _max = limitby

1777 else:

1778 _min, _max = 0, len(self)

1779 count = 0

1780 for i, row in self.records.items():

1781 if f(row):

1782 if _min <= count:

1783 records[i] = row

1784 count += 1

1785 if count == _max:

1786 break

1787

1788 return self.__class__(self, self.model, records)

1789

1790 def exclude(self, f: typing.Callable[[T_MetaInstance], Query]) -> "TypedRows[T_MetaInstance]":

1791 """

1792 Removes elements from the calling Rows object, filtered by the function `f`, \

1793 and returns a new Rows object containing the removed elements.

1794 """

1795 if not self.records:

1796 return self.__class__(self, self.model, {})

1797 removed = {}

1798 to_remove = []

1799 for i in self.records:

1800 row = self[i]

1801 if f(row):

1802 removed[i] = self.records[i]

1803 to_remove.append(i)

1804

1805 [self.records.pop(i) for i in to_remove]

1806

1807 return self.__class__(

1808 self,

1809 self.model,

1810 removed,

1811 )

1812

1813 def sort(self, f: typing.Callable[[T_MetaInstance], Any], reverse: bool = False) -> list[T_MetaInstance]:

1814 """

1815 Returns a list of sorted elements (not sorted in place).

1816 """

1817 return [r for (r, s) in sorted(zip(self.records.values(), self), key=lambda r: f(r[1]), reverse=reverse)]

1818

1819 def __str__(self) -> str:

1820 """

1821 Simple string representation.

1822 """

1823 return f"<TypedRows with {len(self)} records>"

1824

1825 def __repr__(self) -> str:

1826 """

1827 Print a table on repr().

1828 """

1829 data = self.as_dict()

1830 headers = list(next(iter(data.values())).keys())

1831 return mktable(data, headers)

1832

1833 def group_by_value(

1834 self, *fields: "str | Field | TypedField[T]", one_result: bool = False, **kwargs: Any

1835 ) -> dict[T, list[T_MetaInstance]]:

1836 """

1837 Group the rows by a specific field (which will be the dict key).

1838 """

1839 kwargs["one_result"] = one_result

1840 result = super().group_by_value(*fields, **kwargs)

1841 return typing.cast(dict[T, list[T_MetaInstance]], result)

1842

1843 def column(self, column: str = None) -> list[Any]:

1844 """

1845 Get a list of all values in a specific column.

1846

1847 Example:

1848 rows.column('name') -> ['Name 1', 'Name 2', ...]

1849 """

1850 return typing.cast(list[Any], super().column(column))

1851

1852 def as_csv(self) -> str:

1853 """

1854 Dump the data to csv.

1855 """

1856 return typing.cast(str, super().as_csv())

1857

1858 def as_dict(

1859 self,

1860 key: str = None,

1861 compact: bool = False,

1862 storage_to_dict: bool = False,

1863 datetime_to_str: bool = False,

1864 custom_types: list[type] = None,

1865 ) -> dict[int, AnyDict]:

1866 """

1867 Get the data in a dict of dicts.

1868 """

1869 if any([key, compact, storage_to_dict, datetime_to_str, custom_types]):

1870 # functionality not guaranteed

1871 return typing.cast(

1872 dict[int, AnyDict],

1873 super().as_dict(

1874 key or "id",

1875 compact,

1876 storage_to_dict,

1877 datetime_to_str,

1878 custom_types,

1879 ),

1880 )

1881

1882 return {k: v.as_dict() for k, v in self.records.items()}

1883

1884 def as_json(self, default: typing.Callable[[Any], Any] = None, indent: Optional[int] = None, **kwargs: Any) -> str:

1885 """

1886 Turn the data into a dict and then dump to JSON.

1887 """

1888 data = self.as_list()

1889

1890 return as_json.encode(data, default=default, indent=indent, **kwargs)

1891

1892 def json(self, default: typing.Callable[[Any], Any] = None, indent: Optional[int] = None, **kwargs: Any) -> str:

1893 """

1894 Turn the data into a dict and then dump to JSON.

1895 """

1896 return self.as_json(default=default, indent=indent, **kwargs)

1897

1898 def as_list(

1899 self,

1900 compact: bool = False,

1901 storage_to_dict: bool = False,

1902 datetime_to_str: bool = False,

1903 custom_types: list[type] = None,

1904 ) -> list[AnyDict]:

1905 """

1906 Get the data in a list of dicts.

1907 """

1908 if any([compact, storage_to_dict, datetime_to_str, custom_types]):

1909 return typing.cast(list[AnyDict], super().as_list(compact, storage_to_dict, datetime_to_str, custom_types))

1910

1911 return [_.as_dict() for _ in self.records.values()]

1912

1913 def __getitem__(self, item: int) -> T_MetaInstance:

1914 """

1915 You can get a specific row by ID from a typedrows by using rows[idx] notation.

1916

1917 Since pydal's implementation differs (they expect a list instead of a dict with id keys),

1918 using rows[0] will return the first row, regardless of its id.

1919 """

1920 try:

1921 return self.records[item]

1922 except KeyError as e:

1923 if item == 0 and (row := self.first()):

1924 # special case: pydal internals think Rows.records is a list, not a dict

1925 return row

1926

1927 raise e

1928

1929 def get(self, item: int) -> typing.Optional[T_MetaInstance]:

1930 """

1931 Get a row by ID, or receive None if it isn't in this result set.

1932 """

1933 return self.records.get(item)

1934

1935 def update(self, **new_values: Any) -> bool:

1936 """

1937 Update the current rows in the database with new_values.

1938 """

1939 # cast to make mypy understand .id is a TypedField and not an int!

1940 table = typing.cast(typing.Type[TypedTable], self.model._ensure_table_defined())

1941

1942 ids = set(self.column("id"))

1943 query = table.id.belongs(ids)

1944 return bool(self.db(query).update(**new_values))

1945

1946 def delete(self) -> bool:

1947 """

1948 Delete the currently selected rows from the database.

1949 """

1950 # cast to make mypy understand .id is a TypedField and not an int!

1951 table = typing.cast(typing.Type[TypedTable], self.model._ensure_table_defined())

1952

1953 ids = set(self.column("id"))

1954 query = table.id.belongs(ids)

1955 return bool(self.db(query).delete())

1956

1957 def join(

1958 self,

1959 field: "Field | TypedField[Any]",

1960 name: str = None,

1961 constraint: Query = None,

1962 fields: list[str | Field] = None,

1963 orderby: Optional[str | Field] = None,

1964 ) -> T_MetaInstance:

1965 """

1966 This can be used to JOIN with some relationships after the initial select.

1967

1968 Using the querybuilder's .join() method is prefered!

1969 """

1970 result = super().join(field, name, constraint, fields or [], orderby)

1971 return typing.cast(T_MetaInstance, result)

1972

1973 def export_to_csv_file(

1974 self,

1975 ofile: typing.TextIO,

1976 null: Any = "<NULL>",

1977 delimiter: str = ",",

1978 quotechar: str = '"',

1979 quoting: int = csv.QUOTE_MINIMAL,

1980 represent: bool = False,

1981 colnames: list[str] = None,

1982 write_colnames: bool = True,

1983 *args: Any,

1984 **kwargs: Any,

1985 ) -> None:

1986 """

1987 Shadow export_to_csv_file from Rows, but with typing.

1988

1989 See http://web2py.com/books/default/chapter/29/06/the-database-abstraction-layer?search=export_to_csv_file#Exporting-and-importing-data

1990 """

1991 super().export_to_csv_file(

1992 ofile,

1993 null,

1994 *args,

1995 delimiter=delimiter,

1996 quotechar=quotechar,

1997 quoting=quoting,

1998 represent=represent,

1999 colnames=colnames or self.colnames,

2000 write_colnames=write_colnames,

2001 **kwargs,

2002 )

2003

2004 @classmethod

2005 def from_rows(

2006 cls, rows: Rows, model: typing.Type[T_MetaInstance], metadata: Metadata = None

2007 ) -> "TypedRows[T_MetaInstance]":

2008 """

2009 Internal method to convert a Rows object to a TypedRows.

2010 """

2011 return cls(rows, model, metadata=metadata)

2012

2013 def __getstate__(self) -> AnyDict:

2014 """

2015 Used by dill to dump to bytes (exclude db connection etc).

2016 """

2017 return {

2018 "metadata": json.dumps(self.metadata, default=str),

2019 "records": self.records,

2020 "model": str(self.model._table),

2021 "colnames": self.colnames,

2022 }

2023

2024 def __setstate__(self, state: AnyDict) -> None:

2025 """

2026 Used by dill when loading from a bytestring.

2027 """

2028 state["metadata"] = json.loads(state["metadata"])

2029 self.__dict__.update(state)

2030 # db etc. set after undill by caching.py

2031

2032

2033 from .caching import ( # noqa: E402

2034 _remove_cache,

2035 _TypedalCache,

2036 _TypedalCacheDependency,

2037 create_and_hash_cache_key,

2038 get_expire,

2039 load_from_cache,

2040 save_to_cache,

2041)

2042

2043

2044 class QueryBuilder(typing.Generic[T_MetaInstance]):

2045 """

2046 Abstration on top of pydal's query system.

2047 """

2048

2049 model: typing.Type[T_MetaInstance]

2050 query: Query

2051 select_args: list[Any]

2052 select_kwargs: AnyDict

2053 relationships: dict[str, Relationship[Any]]

2054 metadata: Metadata

2055

2056 def __init__(

2057 self,

2058 model: typing.Type[T_MetaInstance],

2059 add_query: Optional[Query] = None,

2060 select_args: Optional[list[Any]] = None,

2061 select_kwargs: Optional[AnyDict] = None,

2062 relationships: dict[str, Relationship[Any]] = None,

2063 metadata: Metadata = None,

2064 ):

2065 """

2066 Normally, you wouldn't manually initialize a QueryBuilder but start using a method on a TypedTable.

2067

2068 Example:

2069 MyTable.where(...) -> QueryBuilder[MyTable]

2070 """

2071 self.model = model

2072 table = model._ensure_table_defined()

2073 default_query = typing.cast(Query, table.id > 0)

2074 self.query = add_query or default_query

2075 self.select_args = select_args or []

2076 self.select_kwargs = select_kwargs or {}

2077 self.relationships = relationships or {}

2078 self.metadata = metadata or {}

2079

2080 def __str__(self) -> str:

2081 """

2082 Simple string representation for the query builder.

2083 """

2084 return f"QueryBuilder for {self.model}"

2085

2086 def __repr__(self) -> str:

2087 """

2088 Advanced string representation for the query builder.

2089 """

2090 return (

2091 f"<QueryBuilder for {self.model} with "

2092 f"{len(self.select_args)} select args; "

2093 f"{len(self.select_kwargs)} select kwargs; "

2094 f"{len(self.relationships)} relationships; "

2095 f"query: {bool(self.query)}; "

2096 f"metadata: {self.metadata}; "

2097 f">"

2098 )

2099

2100 def __bool__(self) -> bool:

2101 """

2102 Querybuilder is truthy if it has rows.

2103 """

2104 return self.count() > 0

2105

2106 def _extend(

2107 self,

2108 add_query: Optional[Query] = None,

2109 overwrite_query: Optional[Query] = None,

2110 select_args: Optional[list[Any]] = None,

2111 select_kwargs: Optional[AnyDict] = None,

2112 relationships: dict[str, Relationship[Any]] = None,

2113 metadata: Metadata = None,

2114 ) -> "QueryBuilder[T_MetaInstance]":

2115 return QueryBuilder(

2116 self.model,

2117 (add_query & self.query) if add_query else overwrite_query or self.query,

2118 (self.select_args + select_args) if select_args else self.select_args,

2119 (self.select_kwargs | select_kwargs) if select_kwargs else self.select_kwargs,

2120 (self.relationships | relationships) if relationships else self.relationships,

2121 (self.metadata | (metadata or {})) if metadata else self.metadata,

2122 )

2123

2124 def select(self, *fields: Any, **options: Any) -> "QueryBuilder[T_MetaInstance]":

2125 """

2126 Fields: database columns by name ('id'), by field reference (table.id) or other (e.g. table.ALL).

2127

2128 Options:

2129 paraphrased from the web2py pydal docs,

2130 For more info, see http://www.web2py.com/books/default/chapter/29/06/the-database-abstraction-layer#orderby-groupby-limitby-distinct-having-orderby_on_limitby-join-left-cache

2131

2132 orderby: field(s) to order by. Supported:

2133 table.name - sort by name, ascending

2134 ~table.name - sort by name, descending

2135 <random> - sort randomly

2136 table.name|table.id - sort by two fields (first name, then id)

2137

2138 groupby, having: together with orderby:

2139 groupby can be a field (e.g. table.name) to group records by

2140 having can be a query, only those `having` the condition are grouped

2141

2142 limitby: tuple of min and max. When using the query builder, .paginate(limit, page) is recommended.

2143 distinct: bool/field. Only select rows that differ

2144 orderby_on_limitby (bool, default: True): by default, an implicit orderby is added when doing limitby.

2145 join: othertable.on(query) - do an INNER JOIN. Using TypeDAL relationships with .join() is recommended!

2146 left: othertable.on(query) - do a LEFT JOIN. Using TypeDAL relationships with .join() is recommended!

2147 cache: cache the query result to speed up repeated queries; e.g. (cache=(cache.ram, 3600), cacheable=True)

2148 """

2149 return self._extend(select_args=list(fields), select_kwargs=options)

2150

2151 def where(

2152 self,

2153 *queries_or_lambdas: Query | typing.Callable[[typing.Type[T_MetaInstance]], Query],

2154 **filters: Any,

2155 ) -> "QueryBuilder[T_MetaInstance]":

2156 """

2157 Extend the builder's query.

2158

2159 Can be used in multiple ways:

2160 .where(Query) -> with a direct query such as `Table.id == 5`

2161 .where(lambda table: table.id == 5) -> with a query via a lambda

2162 .where(id=5) -> via keyword arguments

2163

2164 When using multiple where's, they will be ANDed:

2165 .where(lambda table: table.id == 5).where(lambda table: table.id == 6) == (table.id == 5) & (table.id=6)

2166 When passing multiple queries to a single .where, they will be ORed:

2167 .where(lambda table: table.id == 5, lambda table: table.id == 6) == (table.id == 5) | (table.id=6)

2168 """

2169 new_query = self.query

2170 table = self.model._ensure_table_defined()

2171

2172 for field, value in filters.items():

2173 new_query &= table[field] == value

2174

2175 subquery: DummyQuery | Query = DummyQuery()

2176 for query_or_lambda in queries_or_lambdas:

2177 if isinstance(query_or_lambda, _Query):

2178 subquery |= typing.cast(Query, query_or_lambda)

2179 elif callable(query_or_lambda):

2180 if result := query_or_lambda(self.model):

2181 subquery |= result

2182 elif isinstance(query_or_lambda, (Field, _Field)) or is_typed_field(query_or_lambda):

2183 subquery |= typing.cast(Query, query_or_lambda != None)

2184 else:

2185 raise ValueError(f"Unexpected query type ({type(query_or_lambda)}).")

2186

2187 if subquery:

2188 new_query &= subquery

2189

2190 return self._extend(overwrite_query=new_query)

2191

2192 def join(

2193 self,

2194 *fields: str | typing.Type[TypedTable],

2195 method: JOIN_OPTIONS = None,

2196 on: OnQuery | list[Expression] | Expression = None,

2197 condition: Condition = None,

2198 ) -> "QueryBuilder[T_MetaInstance]":

2199 """

2200 Include relationship fields in the result.

2201

2202 `fields` can be names of Relationships on the current model.

2203 If no fields are passed, all will be used.

2204

2205 By default, the `method` defined in the relationship is used.

2206 This can be overwritten with the `method` keyword argument (left or inner)

2207 """

2208 # todo: allow limiting amount of related rows returned for join?

2209

2210 relationships = self.model.get_relationships()

2211

2212 if condition and on:

2213 raise ValueError("condition and on can not be used together!")

2214 elif condition:

2215 if len(fields) != 1:

2216 raise ValueError("join(field, condition=...) can only be used with exactly one field!")

2217

2218 if isinstance(condition, pydal.objects.Query):

2219 condition = as_lambda(condition)

2220

2221 relationships = {str(fields[0]): relationship(fields[0], condition=condition, join=method)}

2222 elif on:

2223 if len(fields) != 1:

2224 raise ValueError("join(field, on=...) can only be used with exactly one field!")

2225

2226 if isinstance(on, pydal.objects.Expression):

2227 on = [on]

2228

2229 if isinstance(on, list):

2230 on = as_lambda(on)

2231 relationships = {str(fields[0]): relationship(fields[0], on=on, join=method)}

2232

2233 else:

2234 if fields:

2235 # join on every relationship

2236 relationships = {str(k): relationships[str(k)] for k in fields}

2237

2238 if method:

2239 relationships = {str(k): r.clone(join=method) for k, r in relationships.items()}

2240

2241 return self._extend(relationships=relationships)

2242

2243 def cache(

2244 self, *deps: Any, expires_at: Optional[dt.datetime] = None, ttl: Optional[int | dt.timedelta] = None

2245 ) -> "QueryBuilder[T_MetaInstance]":

2246 """

2247 Enable caching for this query to load repeated calls from a dill row \

2248 instead of executing the sql and collecing matching rows again.

2249 """

2250 existing = self.metadata.get("cache", {})

2251

2252 metadata: Metadata = {}

2253

2254 cache_meta = typing.cast(

2255 CacheMetadata,

2256 self.metadata.get("cache", {})

2257 | {

2258 "enabled": True,

2259 "depends_on": existing.get("depends_on", []) + [str(_) for _ in deps],

2260 "expires_at": get_expire(expires_at=expires_at, ttl=ttl),

2261 },

2262 )

2263

2264 metadata["cache"] = cache_meta

2265 return self._extend(metadata=metadata)

2266

2267 def _get_db(self) -> TypeDAL:

2268 if db := self.model._db:

2269 return db

2270 else: # pragma: no cover

2271 raise EnvironmentError("@define or db.define is not called on this class yet!")

2272

2273 def _select_arg_convert(self, arg: Any) -> Any:

2274 # typedfield are not really used at runtime anymore, but leave it in for safety:

2275 if isinstance(arg, TypedField): # pragma: no cover

2276 arg = arg._field

2277

2278 return arg

2279

2280 def delete(self) -> list[int]:

2281 """

2282 Based on the current query, delete rows and return a list of deleted IDs.

2283 """

2284 db = self._get_db()

2285 removed_ids = [_.id for _ in db(self.query).select("id")]

2286 if db(self.query).delete():

2287 # success!

2288 return removed_ids

2289

2290 return []

2291

2292 def _delete(self) -> str:

2293 db = self._get_db()

2294 return str(db(self.query)._delete())

2295

2296 def update(self, **fields: Any) -> list[int]:

2297 """

2298 Based on the current query, update `fields` and return a list of updated IDs.

2299 """

2300 # todo: limit?

2301 db = self._get_db()

2302 updated_ids = db(self.query).select("id").column("id")

2303 if db(self.query).update(**fields):

2304 # success!

2305 return updated_ids

2306

2307 return []

2308

2309 def _update(self, **fields: Any) -> str:

2310 db = self._get_db()

2311 return str(db(self.query)._update(**fields))

2312

2313 def _before_query(self, mut_metadata: Metadata, add_id: bool = True) -> tuple[Query, list[Any], AnyDict]:

2314 select_args = [self._select_arg_convert(_) for _ in self.select_args] or [self.model.ALL]

2315 select_kwargs = self.select_kwargs.copy()

2316 query = self.query

2317 model = self.model

2318 mut_metadata["query"] = query

2319 # require at least id of main table:

2320 select_fields = ", ".join([str(_) for _ in select_args])

2321 tablename = str(model)

2322

2323 if add_id and f"{tablename}.id" not in select_fields:

2324 # fields of other selected, but required ID is missing.

2325 select_args.append(model.id)

2326

2327 if self.relationships:

2328 query, select_args = self._handle_relationships_pre_select(query, select_args, select_kwargs, mut_metadata)

2329

2330 return query, select_args, select_kwargs

2331

2332 def to_sql(self, add_id: bool = False) -> str:

2333 """

2334 Generate the SQL for the built query.

2335 """

2336 db = self._get_db()

2337

2338 query, select_args, select_kwargs = self._before_query({}, add_id=add_id)

2339

2340 return str(db(query)._select(*select_args, **select_kwargs))

2341

2342 def _collect(self) -> str:

2343 """

2344 Alias for to_sql, pydal-like syntax.

2345 """

2346 return self.to_sql()

2347

2348 def _collect_cached(self, metadata: Metadata) -> "TypedRows[T_MetaInstance] | None":

2349 expires_at = metadata["cache"].get("expires_at")

2350 metadata["cache"] |= {

2351 # key is partly dependant on cache metadata but not these:

2352 "key": None,

2353 "status": None,

2354 "cached_at": None,

2355 "expires_at": None,

2356 }

2357

2358 _, key = create_and_hash_cache_key(

2359 self.model,

2360 metadata,

2361 self.query,

2362 self.select_args,

2363 self.select_kwargs,

2364 self.relationships.keys(),

2365 )

2366

2367 # re-set after creating key:

2368 metadata["cache"]["expires_at"] = expires_at

2369 metadata["cache"]["key"] = key

2370

2371 return load_from_cache(key, self._get_db())

2372

2373 def execute(self, add_id: bool = False) -> Rows:

2374 """

2375 Raw version of .collect which only executes the SQL, without performing any magic afterwards.

2376 """

2377 db = self._get_db()

2378 metadata = typing.cast(Metadata, self.metadata.copy())

2379

2380 query, select_args, select_kwargs = self._before_query(metadata, add_id=add_id)

2381

2382 return db(query).select(*select_args, **select_kwargs)

2383

2384 def collect(

2385 self, verbose: bool = False, _to: typing.Type["TypedRows[Any]"] = None, add_id: bool = True

2386 ) -> "TypedRows[T_MetaInstance]":

2387 """

2388 Execute the built query and turn it into model instances, while handling relationships.

2389 """

2390 if _to is None:

2391 _to = TypedRows

2392

2393 db = self._get_db()

2394 metadata = typing.cast(Metadata, self.metadata.copy())

2395

2396 if metadata.get("cache", {}).get("enabled") and (result := self._collect_cached(metadata)):

2397 return result

2398

2399 query, select_args, select_kwargs = self._before_query(metadata, add_id=add_id)

2400

2401 metadata["sql"] = db(query)._select(*select_args, **select_kwargs)

2402

2403 if verbose: # pragma: no cover

2404 print(metadata["sql"])

2405

2406 rows: Rows = db(query).select(*select_args, **select_kwargs)

2407

2408 metadata["final_query"] = str(query)

2409 metadata["final_args"] = [str(_) for _ in select_args]

2410 metadata["final_kwargs"] = select_kwargs

2411

2412 if verbose: # pragma: no cover

2413 print(rows)

2414

2415 if not self.relationships:

2416 # easy

2417 typed_rows = _to.from_rows(rows, self.model, metadata=metadata)

2418

2419 else:

2420 # harder: try to match rows to the belonging objects

2421 # assume structure of {'table': <data>} per row.

2422 # if that's not the case, return default behavior again

2423 typed_rows = self._collect_with_relationships(rows, metadata=metadata, _to=_to)

2424

2425 # only saves if requested in metadata:

2426 return save_to_cache(typed_rows, rows)

2427

2428 def _handle_relationships_pre_select(

2429 self,

2430 query: Query,

2431 select_args: list[Any],

2432 select_kwargs: AnyDict,

2433 metadata: Metadata,

2434 ) -> tuple[Query, list[Any]]:

2435 db = self._get_db()

2436 model = self.model

2437

2438 metadata["relationships"] = set(self.relationships.keys())

2439

2440 # query = self._update_query_for_inner(db, model, query)

2441 join = []

2442 for key, relation in self.relationships.items():

2443 if not relation.condition or relation.join != "inner":

2444 continue

2445

2446 other = relation.get_table(db)

2447 other = other.with_alias(f"{key}_{hash(relation)}")

2448 join.append(other.on(relation.condition(model, other)))

2449

2450 if limitby := select_kwargs.pop("limitby", None):

2451 # if limitby + relationships:

2452 # 1. get IDs of main table entries that match 'query'

2453 # 2. change query to .belongs(id)

2454 # 3. add joins etc

2455

2456 kwargs = {"limitby": limitby}

2457

2458 if join:

2459 kwargs["join"] = join

2460

2461 ids = db(query)._select(model.id, **kwargs)

2462 query = model.id.belongs(ids)

2463 metadata["ids"] = ids

2464

2465 if join:

2466 select_kwargs["join"] = join

2467

2468 left = []

2469

2470 for key, relation in self.relationships.items():

2471 other = relation.get_table(db)

2472 method: JOIN_OPTIONS = relation.join or DEFAULT_JOIN_OPTION

2473

2474 select_fields = ", ".join([str(_) for _ in select_args])

2475 pre_alias = str(other)

2476

2477 if f"{other}." not in select_fields:

2478 # no fields of other selected. add .ALL:

2479 select_args.append(other.ALL)

2480 elif f"{other}.id" not in select_fields:

2481 # fields of other selected, but required ID is missing.

2482 select_args.append(other.id)

2483

2484 if relation.on:

2485 # if it has a .on, it's always a left join!

2486 on = relation.on(model, other)

2487 if not isinstance(on, list): # pragma: no cover

2488 on = [on]

2489

2490 left.extend(on)

2491 elif method == "left":

2492 # .on not given, generate it:

2493 other = other.with_alias(f"{key}_{hash(relation)}")

2494 condition = typing.cast(Query, relation.condition(model, other))

2495 left.append(other.on(condition))

2496 else:

2497 # else: inner join (handled earlier)

2498 other = other.with_alias(f"{key}_{hash(relation)}") # only for replace

2499 # other = other.with_alias(f"{key}_{hash(relation)}")

2500 # query &= relation.condition(model, other)

2501

2502 # if no fields of 'other' are included, add other.ALL

2503 # else: only add other.id if missing

2504 select_fields = ", ".join([str(_) for _ in select_args])

2505

2506 post_alias = str(other).split(" AS ")[-1]

2507 if pre_alias != post_alias:

2508 # replace .select's with aliased:

2509 select_fields = select_fields.replace(

2510 f"{pre_alias}.",

2511 f"{post_alias}.",

2512 )

2513

2514 select_args = select_fields.split(", ")

2515

2516 select_kwargs["left"] = left

2517 return query, select_args

2518

2519 def _collect_with_relationships(

2520 self, rows: Rows, metadata: Metadata, _to: typing.Type["TypedRows[Any]"]

2521 ) -> "TypedRows[T_MetaInstance]":

2522 """

2523 Transform the raw rows into Typed Table model instances.

2524 """

2525 db = self._get_db()

2526 main_table = self.model._ensure_table_defined()

2527

2528 records = {}

2529 seen_relations: dict[str, set[str]] = defaultdict(set) # main id -> set of col + id for relation

2530

2531 for row in rows:

2532 main = row[main_table]

2533 main_id = main.id

2534

2535 if main_id not in records:

2536 records[main_id] = self.model(main)

2537 records[main_id]._with = list(self.relationships.keys())

2538

2539 # setup up all relationship defaults (once)

2540 for col, relationship in self.relationships.items():

2541 records[main_id][col] = [] if relationship.multiple else None

2542

2543 # now add other relationship data

2544 for column, relation in self.relationships.items():

2545 relationship_column = f"{column}_{hash(relation)}"

2546

2547 # relationship_column works for aliases with the same target column.

2548 # if col + relationship not in the row, just use the regular name.

2549

2550 relation_data = (

2551 row[relationship_column] if relationship_column in row else row[relation.get_table_name()]

2552 )

2553

2554 if relation_data.id is None:

2555 # always skip None ids

2556 continue

2557

2558 if f"{column}-{relation_data.id}" in seen_relations[main_id]:

2559 # speed up duplicates

2560 continue

2561 else:

2562 seen_relations[main_id].add(f"{column}-{relation_data.id}")

2563

2564 relation_table = relation.get_table(db)

2565 # hopefully an instance of a typed table and a regular row otherwise:

2566 instance = relation_table(relation_data) if looks_like(relation_table, TypedTable) else relation_data

2567

2568 if relation.multiple:

2569 # create list of T

2570 if not isinstance(records[main_id].get(column), list): # pragma: no cover

2571 # should already be set up before!

2572 setattr(records[main_id], column, [])

2573

2574 records[main_id][column].append(instance)

2575 else:

2576 # create single T

2577 records[main_id][column] = instance

2578

2579 return _to(rows, self.model, records, metadata=metadata)

2580

2581 def collect_or_fail(self, exception: Exception = None) -> "TypedRows[T_MetaInstance]":

2582 """

2583 Call .collect() and raise an error if nothing found.

2584

2585 Basically unwraps Optional type.

2586 """

2587 if result := self.collect():

2588 return result

2589

2590 if not exception:

2591 exception = ValueError("Nothing found!")

2592

2593 raise exception

2594

2595 def __iter__(self) -> typing.Generator[T_MetaInstance, None, None]:

2596 """

2597 You can start iterating a Query Builder object before calling collect, for ease of use.

2598 """

2599 yield from self.collect()

2600

2601 def count(self) -> int:

2602 """

2603 Return the amount of rows matching the current query.

2604 """

2605 db = self._get_db()

2606 model = self.model

2607 query = self.query

2608

2609 for key, relation in self.relationships.items():

2610 if not relation.condition or relation.join != "inner":

2611 continue

2612

2613 other = relation.get_table(db)

2614 other = other.with_alias(f"{key}_{hash(relation)}")

2615 query &= relation.condition(model, other)

2616

2617 return db(query).count()

2618

2619 def __paginate(

2620 self,

2621 limit: int,

2622 page: int = 1,

2623 ) -> "QueryBuilder[T_MetaInstance]":

2624 _from = limit * (page - 1)

2625 _to = limit * page

2626

2627 available = self.count()

2628

2629 metadata: Metadata = {}

2630

2631 metadata["pagination"] = {

2632 "limit": limit,

2633 "current_page": page,

2634 "max_page": math.ceil(available / limit),

2635 "rows": available,

2636 "min_max": (_from, _to),

2637 }

2638

2639 return self._extend(select_kwargs={"limitby": (_from, _to)}, metadata=metadata)

2640

2641 def paginate(self, limit: int, page: int = 1, verbose: bool = False) -> "PaginatedRows[T_MetaInstance]":

2642 """

2643 Paginate transforms the more readable `page` and `limit` to pydals internal limit and offset.

2644

2645 Note: when using relationships, this limit is only applied to the 'main' table and any number of extra rows \

2646 can be loaded with relationship data!

2647 """

2648 builder = self.__paginate(limit, page)

2649

2650 rows = typing.cast(PaginatedRows[T_MetaInstance], builder.collect(verbose=verbose, _to=PaginatedRows))

2651

2652 rows._query_builder = builder

2653 return rows

2654

2655 def _paginate(

2656 self,

2657 limit: int,

2658 page: int = 1,

2659 ) -> str:

2660 builder = self.__paginate(limit, page)

2661 return builder._collect()

2662

2663 def chunk(self, chunk_size: int) -> typing.Generator["TypedRows[T_MetaInstance]", Any, None]:

2664 """

2665 Generator that yields rows from a paginated source in chunks.

2666

2667 This function retrieves rows from a paginated data source in chunks of the

2668 specified `chunk_size` and yields them as TypedRows.

2669

2670 Example:

2671 ```

2672 for chunk_of_rows in Table.where(SomeTable.id > 5).chunk(100):

2673 for row in chunk_of_rows:

2674 # Process each row within the chunk.

2675 pass

2676 ```

2677 """

2678 page = 1

2679

2680 while rows := self.__paginate(chunk_size, page).collect():

2681 yield rows

2682 page += 1

2683

2684 def first(self, verbose: bool = False) -> T_MetaInstance | None:

2685 """

2686 Get the first row matching the currently built query.

2687

2688 Also adds paginate, since it would be a waste to select more rows than needed.

2689 """

2690 if row := self.paginate(page=1, limit=1, verbose=verbose).first():

2691 return self.model.from_row(row)

2692 else:

2693 return None

2694

2695 def _first(self) -> str:

2696 return self._paginate(page=1, limit=1)

2697

2698 def first_or_fail(self, exception: Exception = None, verbose: bool = False) -> T_MetaInstance:

2699 """

2700 Call .first() and raise an error if nothing found.

2701

2702 Basically unwraps Optional type.

2703 """

2704 if inst := self.first(verbose=verbose):

2705 return inst

2706

2707 if not exception:

2708 exception = ValueError("Nothing found!")

2709

2710 raise exception

2711

2712

2713 S = typing.TypeVar("S")

2714

2715

2716 class PaginatedRows(TypedRows[T_MetaInstance]):

2717 """

2718 Extension on top of rows that is used when calling .paginate() instead of .collect().

2719 """

2720

2721 _query_builder: QueryBuilder[T_MetaInstance]

2722

2723 @property

2724 def data(self) -> list[T_MetaInstance]:

2725 """

2726 Get the underlying data.

2727 """

2728 return list(self.records.values())

2729

2730 @property

2731 def pagination(self) -> Pagination:

2732 """

2733 Get all page info.

2734 """

2735 pagination_data = self.metadata["pagination"]

2736

2737 has_next_page = pagination_data["current_page"] < pagination_data["max_page"]

2738 has_prev_page = pagination_data["current_page"] > 1

2739 return {

2740 "total_items": pagination_data["rows"],

2741 "current_page": pagination_data["current_page"],

2742 "per_page": pagination_data["limit"],

2743 "total_pages": pagination_data["max_page"],

2744 "has_next_page": has_next_page,

2745 "has_prev_page": has_prev_page,

2746 "next_page": pagination_data["current_page"] + 1 if has_next_page else None,

2747 "prev_page": pagination_data["current_page"] - 1 if has_prev_page else None,

2748 }

2749

2750 def next(self) -> Self:

2751 """

2752 Get the next page.

2753 """

2754 data = self.metadata["pagination"]

2755 if data["current_page"] >= data["max_page"]:

2756 raise StopIteration("Final Page")

2757

2758 return self._query_builder.paginate(limit=data["limit"], page=data["current_page"] + 1)

2759

2760 def previous(self) -> Self:

2761 """

2762 Get the previous page.

2763 """

2764 data = self.metadata["pagination"]

2765 if data["current_page"] <= 1:

2766 raise StopIteration("First Page")

2767

2768 return self._query_builder.paginate(limit=data["limit"], page=data["current_page"] - 1)

2769

2770 def as_dict(self, *_: Any, **__: Any) -> PaginateDict: # type: ignore

2771 """

2772 Convert to a dictionary with pagination info and original data.

2773

2774 All arguments are ignored!

2775 """

2776 return {"data": super().as_dict(), "pagination": self.pagination}

2777

2778

2779 class TypedSet(pydal.objects.Set): # type: ignore # pragma: no cover

2780 """

2781 Used to make pydal Set more typed.

2782

2783 This class is not actually used, only 'cast' by TypeDAL.__call__

2784 """

2785

2786 def count(self, distinct: bool = None, cache: AnyDict = None) -> int:

2787 """

2788 Count returns an int.

2789 """

2790 result = super().count(distinct, cache)

2791 return typing.cast(int, result)

2792

2793 def select(self, *fields: Any, **attributes: Any) -> TypedRows[T_MetaInstance]:

2794 """

2795 Select returns a TypedRows of a user defined table.

2796

2797 Example:

2798 result: TypedRows[MyTable] = db(MyTable.id > 0).select()

2799

2800 for row in result:

2801 typing.reveal_type(row) # MyTable

2802 """

2803 rows = super().select(*fields, **attributes)

2804 return typing.cast(TypedRows[T_MetaInstance], rows)