Coverage for src/typedal/core.py: 100%

1"""

2 Core functionality of TypeDAL.

3"""

4

5 import contextlib

6 import csv

7 import datetime as dt

8 import inspect

9 import json

10 import math

11 import types

12 import typing

13 import warnings

14 from collections import defaultdict

15 from copy import copy

16 from decimal import Decimal

17 from pathlib import Path

18 from typing import Any, Optional

19

20 import pydal

21 from pydal._globals import DEFAULT

22 from pydal.objects import Field as _Field

23 from pydal.objects import Query as _Query

24 from pydal.objects import Row

25 from pydal.objects import Table as _Table

26 from typing_extensions import Self, Unpack

27

28 from .config import TypeDALConfig, load_config

29 from .helpers import (

30 DummyQuery,

31 all_annotations,

32 all_dict,

33 as_lambda,

34 extract_type_optional,

35 filter_out,

36 instanciate,

37 is_union,

38 looks_like,

39 mktable,

40 origin_is_subclass,

41 to_snake,

42 unwrap_type,

43)

44 from .serializers import as_json

45 from .types import (

46 AfterDeleteCallable,

47 AfterInsertCallable,

48 AfterUpdateCallable,

49 AnyDict,

50 BeforeDeleteCallable,

51 BeforeInsertCallable,

52 BeforeUpdateCallable,

53 CacheMetadata,

54 Expression,

55 Field,

56 Metadata,

57 PaginateDict,

58 Pagination,

59 Query,

60 Rows,

61 SelectKwargs,

62 Table,

63 Validator,

64 _Types,

65)

66

67# use typing.cast(type, ...) to make mypy happy with unions

68 T_annotation = typing.Type[Any] | types.UnionType

69 T_Query = typing.Union["Table", Query, bool, None, "TypedTable", typing.Type["TypedTable"]]

70 T_Value = typing.TypeVar("T_Value") # actual type of the Field (via Generic)

71 T_MetaInstance = typing.TypeVar("T_MetaInstance", bound="TypedTable") # bound="TypedTable"; bound="TableMeta"

72 T = typing.TypeVar("T")

73

74 BASIC_MAPPINGS: dict[T_annotation, str] = {

75 str: "string",

76 int: "integer",

77 bool: "boolean",

78 bytes: "blob",

79 float: "double",

80 object: "json",

81 Decimal: "decimal(10,2)",

82 dt.date: "date",

83 dt.time: "time",

84 dt.datetime: "datetime",

85}

86

87

88 def is_typed_field(cls: Any) -> typing.TypeGuard["TypedField[Any]"]:

89 """

90 Is `cls` an instance or subclass of TypedField?

91

92 Deprecated

93 """

94 return (

95 isinstance(cls, TypedField)

96 or isinstance(typing.get_origin(cls), type)

97 and issubclass(typing.get_origin(cls), TypedField)

98 )

99

100

101 JOIN_OPTIONS = typing.Literal["left", "inner", None]

102 DEFAULT_JOIN_OPTION: JOIN_OPTIONS = "left"

103

104# table-ish paramter:

105 P_Table = typing.Union[typing.Type["TypedTable"], pydal.objects.Table]

106

107 Condition: typing.TypeAlias = typing.Optional[

108 typing.Callable[

109 # self, other -> Query

110 [P_Table, P_Table],

111 Query | bool,

112 ]

113]

114

115 OnQuery: typing.TypeAlias = typing.Optional[

116 typing.Callable[

117 # self, other -> list of .on statements

118 [P_Table, P_Table],

119 list[Expression],

120 ]

121]

122

123 To_Type = typing.TypeVar("To_Type", type[Any], typing.Type[Any], str)

124

125

126 class Relationship(typing.Generic[To_Type]):

127 """

128 Define a relationship to another table.

129 """

130

131 _type: To_Type

132 table: typing.Type["TypedTable"] | type | str

133 condition: Condition

134 on: OnQuery

135 multiple: bool

136 join: JOIN_OPTIONS

137

138 def __init__(

139 self,

140 _type: To_Type,

141 condition: Condition = None,

142 join: JOIN_OPTIONS = None,

143 on: OnQuery = None,

144 ):

145 """

146 Should not be called directly, use relationship() instead!

147 """

148 if condition and on:

149 warnings.warn(f"Relation | Both specified! {condition=} {on=} {_type=}")

150 raise ValueError("Please specify either a condition or an 'on' statement for this relationship!")

151

152 self._type = _type

153 self.condition = condition

154 self.join = "left" if on else join # .on is always left join!

155 self.on = on

156

157 if args := typing.get_args(_type):

158 self.table = unwrap_type(args[0])

159 self.multiple = True

160 else:

161 self.table = _type

162 self.multiple = False

163

164 if isinstance(self.table, str):

165 self.table = TypeDAL.to_snake(self.table)

166

167 def clone(self, **update: Any) -> "Relationship[To_Type]":

168 """

169 Create a copy of the relationship, possibly updated.

170 """

171 return self.__class__(

172 update.get("_type") or self._type,

173 update.get("condition") or self.condition,

174 update.get("join") or self.join,

175 update.get("on") or self.on,

176 )

177

178 def __repr__(self) -> str:

179 """

180 Representation of the relationship.

181 """

182 if callback := self.condition or self.on:

183 src_code = inspect.getsource(callback).strip()

184 else:

185 cls_name = self._type if isinstance(self._type, str) else self._type.__name__ # type: ignore

186 src_code = f"to {cls_name} (missing condition)"

187

188 join = f":{self.join}" if self.join else ""

189 return f"<Relationship{join} {src_code}>"

190

191 def get_table(self, db: "TypeDAL") -> typing.Type["TypedTable"]:

192 """

193 Get the table this relationship is bound to.

194 """

195 table = self.table # can be a string because db wasn't available yet

196 if isinstance(table, str):

197 if mapped := db._class_map.get(table):

198 # yay

199 return mapped

200

201 # boo, fall back to untyped table but pretend it is typed:

202 return typing.cast(typing.Type["TypedTable"], db[table]) # eh close enough!

203

204 return table

205

206 def get_table_name(self) -> str:

207 """

208 Get the name of the table this relationship is bound to.

209 """

210 if isinstance(self.table, str):

211 return self.table

212

213 if isinstance(self.table, pydal.objects.Table):

214 return str(self.table)

215

216 # else: typed table

217 try:

218 table = self.table._ensure_table_defined() if issubclass(self.table, TypedTable) else self.table

219 except Exception: # pragma: no cover

220 table = self.table

221

222 return str(table)

223

224 def __get__(self, instance: Any, owner: Any) -> typing.Optional[list[Any]] | "Relationship[To_Type]":

225 """

226 Relationship is a descriptor class, which can be returned from a class but not an instance.

227

228 For an instance, using .join() will replace the Relationship with the actual data.

229 If you forgot to join, a warning will be shown and empty data will be returned.

230 """

231 if not instance:

232 # relationship queried on class, that's allowed

233 return self

234

235 warnings.warn(

236 "Trying to get data from a relationship object! Did you forget to join it?", category=RuntimeWarning

237 )

238 if self.multiple:

239 return []

240 else:

241 return None

242

243

244 def relationship(

245 _type: To_Type, condition: Condition = None, join: JOIN_OPTIONS = None, on: OnQuery = None

246) -> Relationship[To_Type]:

247 """

248 Define a relationship to another table, when its id is not stored in the current table.

249

250 Example:

251 class User(TypedTable):

252 name: str

253

254 posts = relationship(list["Post"], condition=lambda self, post: self.id == post.author, join='left')

255

256 class Post(TypedTable):

257 title: str

258 author: User

259

260 User.join("posts").first() # User instance with list[Post] in .posts

261

262 Here, Post stores the User ID, but `relationship(list["Post"])` still allows you to get the user's posts.

263 In this case, the join strategy is set to LEFT so users without posts are also still selected.

264

265 For complex queries with a pivot table, a `on` can be set insteaad of `condition`:

266 class User(TypedTable):

267 ...

268

269 tags = relationship(list["Tag"], on=lambda self, tag: [

270 Tagged.on(Tagged.entity == entity.gid),

271 Tag.on((Tagged.tag == tag.id)),

272 ])

273

274 If you'd try to capture this in a single 'condition', pydal would create a cross join which is much less efficient.

275 """

276 return Relationship(_type, condition, join, on)

277

278

279 def _generate_relationship_condition(

280 _: typing.Type["TypedTable"], key: str, field: typing.Union["TypedField[Any]", "Table", typing.Type["TypedTable"]]

281) -> Condition:

282 origin = typing.get_origin(field)

283 # else: generic

284

285 if origin == list:

286 # field = typing.get_args(field)[0] # actual field

287 # return lambda _self, _other: cls[key].contains(field)

288

289 return lambda _self, _other: _self[key].contains(_other.id)

290 else:

291 # normal reference

292 # return lambda _self, _other: cls[key] == field.id

293 return lambda _self, _other: _self[key] == _other.id

294

295

296 def to_relationship(

297 cls: typing.Type["TypedTable"] | type[Any],

298 key: str,

299 field: typing.Union["TypedField[Any]", "Table", typing.Type["TypedTable"]],

300) -> typing.Optional[Relationship[Any]]:

301 """

302 Used to automatically create relationship instance for reference fields.

303

304 Example:

305 class MyTable(TypedTable):

306 reference: OtherTable

307

308 `reference` contains the id of an Other Table row.

309 MyTable.relationships should have 'reference' as a relationship, so `MyTable.join('reference')` should work.

310

311 This function will automatically perform this logic (called in db.define):

312 to_relationship(MyTable, 'reference', OtherTable) -> Relationship[OtherTable]

313

314 Also works for list:reference (list[OtherTable]) and TypedField[OtherTable].

315 """

316 if looks_like(field, TypedField):

317 if args := typing.get_args(field):

318 field = args[0]

319 else:

320 # weird

321 return None

322

323 field, optional = extract_type_optional(field)

324

325 try:

326 condition = _generate_relationship_condition(cls, key, field)

327 except Exception as e: # pragma: no cover

328 warnings.warn("Could not generate Relationship condition", source=e)

329 condition = None

330

331 if not condition: # pragma: no cover

332 # something went wrong, not a valid relationship

333 warnings.warn(f"Invalid relationship for {cls.__name__}.{key}: {field}")

334 return None

335

336 join = "left" if optional or typing.get_origin(field) == list else "inner"

337

338 return Relationship(typing.cast(type[TypedTable], field), condition, typing.cast(JOIN_OPTIONS, join))

339

340

341 class TypeDAL(pydal.DAL): # type: ignore

342 """

343 Drop-in replacement for pyDAL with layer to convert class-based table definitions to classical pydal define_tables.

344 """

345

346 _config: TypeDALConfig

347

348 def __init__(

349 self,

350 uri: Optional[str] = None, # default from config or 'sqlite:memory'

351 pool_size: int = None, # default 1 if sqlite else 3

352 folder: Optional[str | Path] = None, # default 'databases' in config

353 db_codec: str = "UTF-8",

354 check_reserved: Optional[list[str]] = None,

355 migrate: Optional[bool] = None, # default True by config

356 fake_migrate: Optional[bool] = None, # default False by config

357 migrate_enabled: bool = True,

358 fake_migrate_all: bool = False,

359 decode_credentials: bool = False,

360 driver_args: Optional[AnyDict] = None,

361 adapter_args: Optional[AnyDict] = None,

362 attempts: int = 5,

363 auto_import: bool = False,

364 bigint_id: bool = False,

365 debug: bool = False,

366 lazy_tables: bool = False,

367 db_uid: Optional[str] = None,

368 after_connection: typing.Callable[..., Any] = None,

369 tables: Optional[list[str]] = None,

370 ignore_field_case: bool = True,

371 entity_quoting: bool = True,

372 table_hash: Optional[str] = None,

373 enable_typedal_caching: bool = None,

374 use_pyproject: bool | str = True,

375 use_env: bool | str = True,

376 connection: Optional[str] = None,

377 config: Optional[TypeDALConfig] = None,

378 ) -> None:

379 """

380 Adds some internal tables after calling pydal's default init.

381

382 Set enable_typedal_caching to False to disable this behavior.

383 """

384 config = config or load_config(connection, _use_pyproject=use_pyproject, _use_env=use_env)

385 config.update(

386 database=uri,

387 dialect=uri.split(":")[0] if uri and ":" in uri else None,

388 folder=str(folder) if folder is not None else None,

389 migrate=migrate,

390 fake_migrate=fake_migrate,

391 caching=enable_typedal_caching,

392 pool_size=pool_size,

393 )

394

395 self._config = config

396

397 if config.folder:

398 Path(config.folder).mkdir(exist_ok=True)

399

400 super().__init__(

401 config.database,

402 config.pool_size,

403 config.folder,

404 db_codec,

405 check_reserved,

406 config.migrate,

407 config.fake_migrate,

408 migrate_enabled,

409 fake_migrate_all,

410 decode_credentials,

411 driver_args,

412 adapter_args,

413 attempts,

414 auto_import,

415 bigint_id,

416 debug,

417 lazy_tables,

418 db_uid,

419 after_connection,

420 tables,

421 ignore_field_case,

422 entity_quoting,

423 table_hash,

424 )

425

426 if config.caching:

427 self.try_define(_TypedalCache)

428 self.try_define(_TypedalCacheDependency)

429

430 def try_define(self, model: typing.Type[T], verbose: bool = False) -> typing.Type[T]:

431 """

432 Try to define a model with migrate or fall back to fake migrate.

433 """

434 try:

435 return self.define(model, migrate=True)

436 except Exception as e:

437 # clean up:

438 self.rollback()

439 if (tablename := self.to_snake(model.__name__)) and tablename in dir(self):

440 delattr(self, tablename)

441

442 if verbose:

443 warnings.warn(f"{model} could not be migrated, try faking", source=e, category=RuntimeWarning)

444

445 # try again:

446 return self.define(model, migrate=True, fake_migrate=True, redefine=True)

447

448 default_kwargs: typing.ClassVar[AnyDict] = {

449 # fields are 'required' (notnull) by default:

450 "notnull": True,

451 }

452

453 # maps table name to typedal class, for resolving future references

454 _class_map: typing.ClassVar[dict[str, typing.Type["TypedTable"]]] = {}

455

456 def _define(self, cls: typing.Type[T], **kwargs: Any) -> typing.Type[T]:

457 # todo: new relationship item added should also invalidate (previously unrelated) cache result

458

459 # todo: option to enable/disable cache dependency behavior:

460 # - don't set _before_update and _before_delete

461 # - don't add TypedalCacheDependency entry

462 # - don't invalidate other item on new row of this type

463

464 # when __future__.annotations is implemented, cls.__annotations__ will not work anymore as below.

465 # proper way to handle this would be (but gives error right now due to Table implementing magic methods):

466 # typing.get_type_hints(cls, globalns=None, localns=None)

467

468 # dirty way (with evil eval):

469 # [eval(v) for k, v in cls.__annotations__.items()]

470 # this however also stops working when variables outside this scope or even references to other

471 # objects are used. So for now, this package will NOT work when from __future__ import annotations is used,

472 # and might break in the future, when this annotations behavior is enabled by default.

473

474 # non-annotated variables have to be passed to define_table as kwargs

475 full_dict = all_dict(cls) # includes properties from parents (e.g. useful for mixins)

476

477 tablename = self.to_snake(cls.__name__)

478 # grab annotations of cls and it's parents:

479 annotations = all_annotations(cls)

480 # extend with `prop = TypedField()` 'annotations':

481 annotations |= {k: typing.cast(type, v) for k, v in full_dict.items() if is_typed_field(v)}

482 # remove internal stuff:

483 annotations = {k: v for k, v in annotations.items() if not k.startswith("_")}

484

485 typedfields: dict[str, TypedField[Any]] = {

486 k: instanciate(v, True) for k, v in annotations.items() if is_typed_field(v)

487 }

488

489 relationships: dict[str, type[Relationship[Any]]] = filter_out(annotations, Relationship)

490

491 fields = {fname: self._to_field(fname, ftype) for fname, ftype in annotations.items()}

492

493 # ! dont' use full_dict here:

494 other_kwargs = kwargs | {

495 k: v for k, v in cls.__dict__.items() if k not in annotations and not k.startswith("_")

496 } # other_kwargs was previously used to pass kwargs to typedal, but use @define(**kwargs) for that.

497 # now it's only used to extract relationships from the object.

498 # other properties of the class (incl methods) should not be touched

499

500 # for key in typedfields.keys() - full_dict.keys():

501 # # typed fields that don't haven't been added to the object yet

502 # setattr(cls, key, typedfields[key])

503

504 for key, field in typedfields.items():

505 # clone every property so it can be re-used across mixins:

506 clone = copy(field)

507 setattr(cls, key, clone)

508 typedfields[key] = clone

509

510 # start with base classes and overwrite with current class:

511 relationships = filter_out(full_dict, Relationship) | relationships | filter_out(other_kwargs, Relationship)

512

513 # DEPRECATED: Relationship as annotation is currently not supported!

514 # ensure they are all instances and

515 # not mix of instances (`= relationship()`) and classes (`: Relationship[...]`):

516 # relationships = {

517 # k: v if isinstance(v, Relationship) else to_relationship(cls, k, v) for k, v in relationships.items()

518 # }

519

520 # keys of implicit references (also relationships):

521 reference_field_keys = [k for k, v in fields.items() if v.type.split(" ")[0] in ("list:reference", "reference")]

522

523 # add implicit relationships:

524 # User; list[User]; TypedField[User]; TypedField[list[User]]

525 relationships |= {

526 k: new_relationship

527 for k in reference_field_keys

528 if k not in relationships and (new_relationship := to_relationship(cls, k, annotations[k]))

529 }

530

531 cache_dependency = self._config.caching and kwargs.pop("cache_dependency", True)

532

533 table: Table = self.define_table(tablename, *fields.values(), **kwargs)

534

535 for name, typed_field in typedfields.items():

536 field = fields[name]

537 typed_field.bind(field, table)

538

539 if issubclass(cls, TypedTable):

540 cls.__set_internals__(

541 db=self,

542 table=table,

543 # by now, all relationships should be instances!

544 relationships=typing.cast(dict[str, Relationship[Any]], relationships),

545 )

546 self._class_map[str(table)] = cls

547 cls.__on_define__(self)

548 else:

549 warnings.warn("db.define used without inheriting TypedTable. This could lead to strange problems!")

550

551 if not tablename.startswith("typedal_") and cache_dependency:

552 table._before_update.append(lambda s, _: _remove_cache(s, tablename))

553 table._before_delete.append(lambda s: _remove_cache(s, tablename))

554

555 return cls

556

557 @typing.overload

558 def define(self, maybe_cls: None = None, **kwargs: Any) -> typing.Callable[[typing.Type[T]], typing.Type[T]]:

559 """

560 Typing Overload for define without a class.

561

562 @db.define()

563 class MyTable(TypedTable): ...

564 """

565

566 @typing.overload

567 def define(self, maybe_cls: typing.Type[T], **kwargs: Any) -> typing.Type[T]:

568 """

569 Typing Overload for define with a class.

570

571 @db.define

572 class MyTable(TypedTable): ...

573 """

574

575 def define(

576 self, maybe_cls: typing.Type[T] | None = None, **kwargs: Any

577 ) -> typing.Type[T] | typing.Callable[[typing.Type[T]], typing.Type[T]]:

578 """

579 Can be used as a decorator on a class that inherits `TypedTable`, \

580 or as a regular method if you need to define your classes before you have access to a 'db' instance.

581

582 You can also pass extra arguments to db.define_table.

583 See http://www.web2py.com/books/default/chapter/29/06/the-database-abstraction-layer#Table-constructor

584

585 Example:

586 @db.define

587 class Person(TypedTable):

588 ...

589

590 class Article(TypedTable):

591 ...

592

593 # at a later time:

594 db.define(Article)

595

596 Returns:

597 the result of pydal.define_table

598 """

599

600 def wrapper(cls: typing.Type[T]) -> typing.Type[T]:

601 return self._define(cls, **kwargs)

602

603 if maybe_cls:

604 return wrapper(maybe_cls)

605

606 return wrapper

607

608 # def drop(self, table_name: str) -> None:

609 # """

610 # Remove a table by name (both on the database level and the typedal level).

611 # """

612 # # drop calls TypedTable.drop() and removes it from the `_class_map`

613 # if cls := self._class_map.pop(table_name, None):

614 # cls.drop()

615

616 # def drop_all(self, max_retries: int = None) -> None:

617 # """

618 # Remove all tables and keep doing so until everything is gone!

619 # """

620 # retries = 0

621 # if max_retries is None:

622 # max_retries = len(self.tables)

623 #

624 # while self.tables:

625 # retries += 1

626 # for table in self.tables:

627 # self.drop(table)

628 #

629 # if retries > max_retries:

630 # raise RuntimeError("Could not delete all tables")

631

632 def __call__(self, *_args: T_Query, **kwargs: Any) -> "TypedSet":

633 """

634 A db instance can be called directly to perform a query.

635

636 Usually, only a query is passed.

637

638 Example:

639 db(query).select()

640

641 """

642 args = list(_args)

643 if args:

644 cls = args[0]

645 if isinstance(cls, bool):

646 raise ValueError("Don't actually pass a bool to db()! Use a query instead.")

647

648 if isinstance(cls, type) and issubclass(type(cls), type) and issubclass(cls, TypedTable):

649 # table defined without @db.define decorator!

650 _cls: typing.Type[TypedTable] = cls

651 args[0] = _cls.id != None

652

653 _set = super().__call__(*args, **kwargs)

654 return typing.cast(TypedSet, _set)

655

656 def __getitem__(self, key: str) -> "Table":

657 """

658 Allows dynamically accessing a table by its name as a string.

659

660 Example:

661 db['users'] -> user

662 """

663 return typing.cast(Table, super().__getitem__(str(key)))

664

665 @classmethod

666 def _build_field(cls, name: str, _type: str, **kw: Any) -> Field:

667 return Field(name, _type, **{**cls.default_kwargs, **kw})

668

669 @classmethod

670 def _annotation_to_pydal_fieldtype(

671 cls, _ftype: T_annotation, mut_kw: typing.MutableMapping[str, Any]

672 ) -> Optional[str]:

673 # ftype can be a union or type. typing.cast is sometimes used to tell mypy when it's not a union.

674 ftype = typing.cast(type, _ftype) # cast from typing.Type to type to make mypy happy)

675

676 if isinstance(ftype, str):

677 # extract type from string

678 ftype = typing.get_args(typing.Type[ftype])[0]._evaluate(

679 localns=locals(), globalns=globals(), recursive_guard=frozenset()

680 )

681

682 if mapping := BASIC_MAPPINGS.get(ftype):

683 # basi types

684 return mapping

685 elif isinstance(ftype, _Table):

686 # db.table

687 return f"reference {ftype._tablename}"

688 elif issubclass(type(ftype), type) and issubclass(ftype, TypedTable):

689 # SomeTable

690 snakename = cls.to_snake(ftype.__name__)

691 return f"reference {snakename}"

692 elif isinstance(ftype, TypedField):

693 # FieldType(type, ...)

694 return ftype._to_field(mut_kw)

695 elif origin_is_subclass(ftype, TypedField):

696 # TypedField[int]

697 return cls._annotation_to_pydal_fieldtype(typing.get_args(ftype)[0], mut_kw)

698 elif isinstance(ftype, types.GenericAlias) and typing.get_origin(ftype) in (list, TypedField):

699 # list[str] -> str -> string -> list:string

700 _child_type = typing.get_args(ftype)[0]

701 _child_type = cls._annotation_to_pydal_fieldtype(_child_type, mut_kw)

702 return f"list:{_child_type}"

703 elif is_union(ftype):

704 # str | int -> UnionType

705 # typing.Union[str | int] -> typing._UnionGenericAlias

706

707 # Optional[type] == type | None

708

709 match typing.get_args(ftype):

710 case (_child_type, _Types.NONETYPE) | (_Types.NONETYPE, _child_type):

711 # good union of Nullable

712

713 # if a field is optional, it is nullable:

714 mut_kw["notnull"] = False

715 return cls._annotation_to_pydal_fieldtype(_child_type, mut_kw)

716 case _:

717 # two types is not supported by the db!

718 return None

719 else:

720 return None

721

722 @classmethod

723 def _to_field(cls, fname: str, ftype: type, **kw: Any) -> Field:

724 """

725 Convert a annotation into a pydal Field.

726

727 Args:

728 fname: name of the property

729 ftype: annotation of the property

730 kw: when using TypedField or a function returning it (e.g. StringField),

731 keyword args can be used to pass any other settings you would normally to a pydal Field

732

733 -> pydal.Field(fname, ftype, **kw)

734

735 Example:

736 class MyTable:

737 fname: ftype

738 id: int

739 name: str

740 reference: Table

741 other: TypedField(str, default="John Doe") # default will be in kwargs

742 """

743 fname = cls.to_snake(fname)

744

745 if converted_type := cls._annotation_to_pydal_fieldtype(ftype, kw):

746 return cls._build_field(fname, converted_type, **kw)

747 else:

748 raise NotImplementedError(f"Unsupported type {ftype}/{type(ftype)}")

749

750 @staticmethod

751 def to_snake(camel: str) -> str:

752 """

753 Moved to helpers, kept as a static method for legacy reasons.

754 """

755 return to_snake(camel)

756

757

758 class TableMeta(type):

759 """

760 This metaclass contains functionality on table classes, that doesn't exist on its instances.

761

762 Example:

763 class MyTable(TypedTable):

764 some_field: TypedField[int]

765

766 MyTable.update_or_insert(...) # should work

767

768 MyTable.some_field # -> Field, can be used to query etc.

769

770 row = MyTable.first() # returns instance of MyTable

771

772 # row.update_or_insert(...) # shouldn't work!

773

774 row.some_field # -> int, with actual data

775

776 """

777

778 # set up by db.define:

779 # _db: TypeDAL | None = None

780 # _table: Table | None = None

781 _db: TypeDAL | None = None

782 _table: Table | None = None

783 _relationships: dict[str, Relationship[Any]] | None = None

784

785 #########################

786 # TypeDAL custom logic: #

787 #########################

788

789 def __set_internals__(self, db: pydal.DAL, table: Table, relationships: dict[str, Relationship[Any]]) -> None:

790 """

791 Store the related database and pydal table for later usage.

792 """

793 self._db = db

794 self._table = table

795 self._relationships = relationships

796

797 def __getattr__(self, col: str) -> Optional[Field]:

798 """

799 Magic method used by TypedTableMeta to get a database field with dot notation on a class.

800

801 Example:

802 SomeTypedTable.col -> db.table.col (via TypedTableMeta.__getattr__)

803

804 """

805 if self._table:

806 return getattr(self._table, col, None)

807

808 return None

809

810 def _ensure_table_defined(self) -> Table:

811 if not self._table:

812 raise EnvironmentError("@define or db.define is not called on this class yet!")

813 return self._table

814

815 def __iter__(self) -> typing.Generator[Field, None, None]:

816 """

817 Loop through the columns of this model.

818 """

819 table = self._ensure_table_defined()

820 yield from iter(table)

821

822 def __getitem__(self, item: str) -> Field:

823 """

824 Allow dict notation to get a column of this table (-> Field instance).

825 """

826 table = self._ensure_table_defined()

827 return table[item]

828

829 def __str__(self) -> str:

830 """

831 Normally, just returns the underlying table name, but with a fallback if the model is unbound.

832 """

833 if self._table:

834 return str(self._table)

835 else:

836 return f"<unbound table {self.__name__}>"

837

838 def from_row(self: typing.Type[T_MetaInstance], row: pydal.objects.Row) -> T_MetaInstance:

839 """

840 Create a model instance from a pydal row.

841 """

842 return self(row)

843

844 def all(self: typing.Type[T_MetaInstance]) -> "TypedRows[T_MetaInstance]":

845 """

846 Return all rows for this model.

847 """

848 return self.collect()

849

850 def get_relationships(self) -> dict[str, Relationship[Any]]:

851 """

852 Return the registered relationships of the current model.

853 """

854 return self._relationships or {}

855

856 ##########################

857 # TypeDAL Modified Logic #

858 ##########################

859

860 def insert(self: typing.Type[T_MetaInstance], **fields: Any) -> T_MetaInstance:

861 """

862 This is only called when db.define is not used as a decorator.

863

864 cls.__table functions as 'self'

865

866 Args:

867 **fields: anything you want to insert in the database

868

869 Returns: the ID of the new row.

870

871 """

872 table = self._ensure_table_defined()

873

874 result = table.insert(**fields)

875 # it already is an int but mypy doesn't understand that

876 return self(result)

877

878 def _insert(self, **fields: Any) -> str:

879 table = self._ensure_table_defined()

880

881 return str(table._insert(**fields))

882

883 def bulk_insert(self: typing.Type[T_MetaInstance], items: list[AnyDict]) -> "TypedRows[T_MetaInstance]":

884 """

885 Insert multiple rows, returns a TypedRows set of new instances.

886 """

887 table = self._ensure_table_defined()

888 result = table.bulk_insert(items)

889 return self.where(lambda row: row.id.belongs(result)).collect()

890

891 def update_or_insert(

892 self: typing.Type[T_MetaInstance], query: T_Query | AnyDict = DEFAULT, **values: Any

893 ) -> T_MetaInstance:

894 """

895 Update a row if query matches, else insert a new one.

896

897 Returns the created or updated instance.

898 """

899 table = self._ensure_table_defined()

900

901 if query is DEFAULT:

902 record = table(**values)

903 elif isinstance(query, dict):

904 record = table(**query)

905 else:

906 record = table(query)

907

908 if not record:

909 return self.insert(**values)

910

911 record.update_record(**values)

912 return self(record)

913

914 def validate_and_insert(

915 self: typing.Type[T_MetaInstance], **fields: Any

916 ) -> tuple[Optional[T_MetaInstance], Optional[dict[str, str]]]:

917 """

918 Validate input data and then insert a row.

919

920 Returns a tuple of (the created instance, a dict of errors).

921 """

922 table = self._ensure_table_defined()

923 result = table.validate_and_insert(**fields)

924 if row_id := result.get("id"):

925 return self(row_id), None

926 else:

927 return None, result.get("errors")

928

929 def validate_and_update(

930 self: typing.Type[T_MetaInstance], query: Query, **fields: Any

931 ) -> tuple[Optional[T_MetaInstance], Optional[dict[str, str]]]:

932 """

933 Validate input data and then update max 1 row.

934

935 Returns a tuple of (the updated instance, a dict of errors).

936 """

937 table = self._ensure_table_defined()

938

939 result = table.validate_and_update(query, **fields)

940

941 if errors := result.get("errors"):

942 return None, errors

943 elif row_id := result.get("id"):

944 return self(row_id), None

945 else: # pragma: no cover

946 # update on query without result (shouldnt happen)

947 return None, None

948

949 def validate_and_update_or_insert(

950 self: typing.Type[T_MetaInstance], query: Query, **fields: Any

951 ) -> tuple[Optional[T_MetaInstance], Optional[dict[str, str]]]:

952 """

953 Validate input data and then update_and_insert (on max 1 row).

954

955 Returns a tuple of (the updated/created instance, a dict of errors).

956 """

957 table = self._ensure_table_defined()

958 result = table.validate_and_update_or_insert(query, **fields)

959

960 if errors := result.get("errors"):

961 return None, errors

962 elif row_id := result.get("id"):

963 return self(row_id), None

964 else: # pragma: no cover

965 # update on query without result (shouldnt happen)

966 return None, None

967

968 def select(self: typing.Type[T_MetaInstance], *a: Any, **kw: Any) -> "QueryBuilder[T_MetaInstance]":

969 """

970 See QueryBuilder.select!

971 """

972 return QueryBuilder(self).select(*a, **kw)

973

974 def paginate(self: typing.Type[T_MetaInstance], limit: int, page: int = 1) -> "PaginatedRows[T_MetaInstance]":

975 """

976 See QueryBuilder.paginate!

977 """

978 return QueryBuilder(self).paginate(limit=limit, page=page)

979

980 def chunk(

981 self: typing.Type[T_MetaInstance], chunk_size: int

982 ) -> typing.Generator["TypedRows[T_MetaInstance]", Any, None]:

983 """

984 See QueryBuilder.chunk!

985 """

986 return QueryBuilder(self).chunk(chunk_size)

987

988 def where(self: typing.Type[T_MetaInstance], *a: Any, **kw: Any) -> "QueryBuilder[T_MetaInstance]":

989 """

990 See QueryBuilder.where!

991 """

992 return QueryBuilder(self).where(*a, **kw)

993

994 def cache(self: typing.Type[T_MetaInstance], *deps: Any, **kwargs: Any) -> "QueryBuilder[T_MetaInstance]":

995 """

996 See QueryBuilder.cache!

997 """

998 return QueryBuilder(self).cache(*deps, **kwargs)

999

1000 def count(self: typing.Type[T_MetaInstance]) -> int:

1001 """

1002 See QueryBuilder.count!

1003 """

1004 return QueryBuilder(self).count()

1005

1006 def first(self: typing.Type[T_MetaInstance]) -> T_MetaInstance | None:

1007 """

1008 See QueryBuilder.first!

1009 """

1010 return QueryBuilder(self).first()

1011

1012 def first_or_fail(self: typing.Type[T_MetaInstance]) -> T_MetaInstance:

1013 """

1014 See QueryBuilder.first_or_fail!

1015 """

1016 return QueryBuilder(self).first_or_fail()

1017

1018 def join(

1019 self: typing.Type[T_MetaInstance],

1020 *fields: str | typing.Type["TypedTable"],

1021 method: JOIN_OPTIONS = None,

1022 on: OnQuery | list[Expression] | Expression = None,

1023 condition: Condition = None,

1024 ) -> "QueryBuilder[T_MetaInstance]":

1025 """

1026 See QueryBuilder.join!

1027 """

1028 return QueryBuilder(self).join(*fields, on=on, condition=condition, method=method)

1029

1030 def collect(self: typing.Type[T_MetaInstance], verbose: bool = False) -> "TypedRows[T_MetaInstance]":

1031 """

1032 See QueryBuilder.collect!

1033 """

1034 return QueryBuilder(self).collect(verbose=verbose)

1035

1036 @property

1037 def ALL(cls) -> pydal.objects.SQLALL:

1038 """

1039 Select all fields for this table.

1040 """

1041 table = cls._ensure_table_defined()

1042

1043 return table.ALL

1044

1045 ##########################

1046 # TypeDAL Shadowed Logic #

1047 ##########################

1048 fields: list[str]

1049

1050 # other table methods:

1051

1052 def truncate(self, mode: str = "") -> None:

1053 """

1054 Remove all data and reset index.

1055 """

1056 table = self._ensure_table_defined()

1057 table.truncate(mode)

1058

1059 def drop(self, mode: str = "") -> None:

1060 """

1061 Remove the underlying table.

1062 """

1063 table = self._ensure_table_defined()

1064 table.drop(mode)

1065

1066 def create_index(self, name: str, *fields: Field | str, **kwargs: Any) -> bool:

1067 """

1068 Add an index on some columns of this table.

1069 """

1070 table = self._ensure_table_defined()

1071 result = table.create_index(name, *fields, **kwargs)

1072 return typing.cast(bool, result)

1073

1074 def drop_index(self, name: str, if_exists: bool = False) -> bool:

1075 """

1076 Remove an index from this table.

1077 """

1078 table = self._ensure_table_defined()

1079 result = table.drop_index(name, if_exists)

1080 return typing.cast(bool, result)

1081

1082 def import_from_csv_file(

1083 self,

1084 csvfile: typing.TextIO,

1085 id_map: dict[str, str] = None,

1086 null: Any = "<NULL>",

1087 unique: str = "uuid",

1088 id_offset: dict[str, int] = None, # id_offset used only when id_map is None

1089 transform: typing.Callable[[dict[Any, Any]], dict[Any, Any]] = None,

1090 validate: bool = False,

1091 encoding: str = "utf-8",

1092 delimiter: str = ",",

1093 quotechar: str = '"',

1094 quoting: int = csv.QUOTE_MINIMAL,

1095 restore: bool = False,

1096 **kwargs: Any,

1097 ) -> None:

1098 """

1099 Load a csv file into the database.

1100 """

1101 table = self._ensure_table_defined()

1102 table.import_from_csv_file(

1103 csvfile,

1104 id_map=id_map,

1105 null=null,

1106 unique=unique,

1107 id_offset=id_offset,

1108 transform=transform,

1109 validate=validate,

1110 encoding=encoding,

1111 delimiter=delimiter,

1112 quotechar=quotechar,

1113 quoting=quoting,

1114 restore=restore,

1115 **kwargs,

1116 )

1117

1118 def on(self, query: Query | bool) -> Expression:

1119 """

1120 Shadow Table.on.

1121

1122 Used for joins.

1123

1124 See Also:

1125 http://web2py.com/books/default/chapter/29/06/the-database-abstraction-layer?search=export_to_csv_file#One-to-many-relation

1126 """

1127 table = self._ensure_table_defined()

1128 return typing.cast(Expression, table.on(query))

1129

1130 def with_alias(self, alias: str) -> _Table:

1131 """

1132 Shadow Table.with_alias.

1133

1134 Useful for joins when joining the same table multiple times.

1135

1136 See Also:

1137 http://web2py.com/books/default/chapter/29/06/the-database-abstraction-layer?search=export_to_csv_file#One-to-many-relation

1138 """

1139 table = self._ensure_table_defined()

1140 return table.with_alias(alias)

1141

1142 # @typing.dataclass_transform()

1143

1144

1145 class TypedField(Expression, typing.Generic[T_Value]): # pragma: no cover

1146 """

1147 Typed version of pydal.Field, which will be converted to a normal Field in the background.

1148 """

1149

1150 # will be set by .bind on db.define

1151 name = ""

1152 _db: Optional[pydal.DAL] = None

1153 _rname: Optional[str] = None

1154 _table: Optional[Table] = None

1155 _field: Optional[Field] = None

1156

1157 _type: T_annotation

1158 kwargs: Any

1159

1160 requires: Validator | typing.Iterable[Validator]

1161

1162 def __init__(self, _type: typing.Type[T_Value] | types.UnionType = str, /, **settings: Any) -> None: # type: ignore

1163 """

1164 A TypedFieldType should not be inited manually, but TypedField (from `fields.py`) should be used!

1165 """

1166 self._type = _type

1167 self.kwargs = settings

1168 # super().__init__()

1169

1170 @typing.overload

1171 def __get__(self, instance: T_MetaInstance, owner: typing.Type[T_MetaInstance]) -> T_Value: # pragma: no cover

1172 """

1173 row.field -> (actual data).

1174 """

1175

1176 @typing.overload

1177 def __get__(self, instance: None, owner: "typing.Type[TypedTable]") -> "TypedField[T_Value]": # pragma: no cover

1178 """

1179 Table.field -> Field.

1180 """

1181

1182 def __get__(

1183 self, instance: T_MetaInstance | None, owner: typing.Type[T_MetaInstance]

1184 ) -> typing.Union[T_Value, "TypedField[T_Value]"]:

1185 """

1186 Since this class is a Descriptor field, \

1187 it returns something else depending on if it's called on a class or instance.

1188

1189 (this is mostly for mypy/typing)

1190 """

1191 if instance:

1192 # this is only reached in a very specific case:

1193 # an instance of the object was created with a specific set of fields selected (excluding the current one)

1194 # in that case, no value was stored in the owner -> return None (since the field was not selected)

1195 return typing.cast(T_Value, None) # cast as T_Value so mypy understands it for selected fields

1196 else:

1197 # getting as class -> return actual field so pydal understands it when using in query etc.

1198 return typing.cast(TypedField[T_Value], self._field) # pretend it's still typed for IDE support

1199

1200 def __str__(self) -> str:

1201 """

1202 String representation of a Typed Field.

1203

1204 If `type` is set explicitly (e.g. TypedField(str, type="text")), that type is used: `TypedField.text`,

1205 otherwise the type annotation is used (e.g. TypedField(str) -> TypedField.str)

1206 """

1207 return str(self._field) if self._field else ""

1208

1209 def __repr__(self) -> str:

1210 """

1211 More detailed string representation of a Typed Field.

1212

1213 Uses __str__ and adds the provided extra options (kwargs) in the representation.

1214 """

1215 s = self.__str__()

1216

1217 if "type" in self.kwargs:

1218 # manual type in kwargs supplied

1219 t = self.kwargs["type"]

1220 elif issubclass(type, type(self._type)):

1221 # normal type, str.__name__ = 'str'

1222 t = getattr(self._type, "__name__", str(self._type))

1223 elif t_args := typing.get_args(self._type):

1224 # list[str] -> 'str'

1225 t = t_args[0].__name__

1226 else: # pragma: no cover

1227 # fallback - something else, may not even happen, I'm not sure

1228 t = self._type

1229

1230 s = f"TypedField[{t}].{s}" if s else f"TypedField[{t}]"

1231

1232 kw = self.kwargs.copy()

1233 kw.pop("type", None)

1234 return f"<{s} with options {kw}>"

1235

1236 def _to_field(self, extra_kwargs: typing.MutableMapping[str, Any]) -> Optional[str]:

1237 """

1238 Convert a Typed Field instance to a pydal.Field.

1239 """

1240 other_kwargs = self.kwargs.copy()

1241 extra_kwargs.update(other_kwargs)

1242 return extra_kwargs.pop("type", False) or TypeDAL._annotation_to_pydal_fieldtype(self._type, extra_kwargs)

1243

1244 def bind(self, field: pydal.objects.Field, table: pydal.objects.Table) -> None:

1245 """

1246 Bind the right db/table/field info to this class, so queries can be made using `Class.field == ...`.

1247 """

1248 self._table = table

1249 self._field = field

1250

1251 def __getattr__(self, key: str) -> Any:

1252 """

1253 If the regular getattribute does not work, try to get info from the related Field.

1254 """

1255 with contextlib.suppress(AttributeError):

1256 return super().__getattribute__(key)

1257

1258 # try on actual field:

1259 return getattr(self._field, key)

1260

1261 def __eq__(self, other: Any) -> Query:

1262 """

1263 Performing == on a Field will result in a Query.

1264 """

1265 return typing.cast(Query, self._field == other)

1266

1267 def __ne__(self, other: Any) -> Query:

1268 """

1269 Performing != on a Field will result in a Query.

1270 """

1271 return typing.cast(Query, self._field != other)

1272

1273 def __gt__(self, other: Any) -> Query:

1274 """

1275 Performing > on a Field will result in a Query.

1276 """

1277 return typing.cast(Query, self._field > other)

1278

1279 def __lt__(self, other: Any) -> Query:

1280 """

1281 Performing < on a Field will result in a Query.

1282 """

1283 return typing.cast(Query, self._field < other)

1284

1285 def __ge__(self, other: Any) -> Query:

1286 """

1287 Performing >= on a Field will result in a Query.

1288 """

1289 return typing.cast(Query, self._field >= other)

1290

1291 def __le__(self, other: Any) -> Query:

1292 """

1293 Performing <= on a Field will result in a Query.

1294 """

1295 return typing.cast(Query, self._field <= other)

1296

1297 def __hash__(self) -> int:

1298 """

1299 Shadow Field.__hash__.

1300 """

1301 return hash(self._field)

1302

1303 def __invert__(self) -> Expression:

1304 """

1305 Performing ~ on a Field will result in an Expression.

1306 """

1307 if not self._field: # pragma: no cover

1308 raise ValueError("Unbound Field can not be inverted!")

1309

1310 return typing.cast(Expression, ~self._field)

1311

1312 def lower(self) -> Expression:

1313 """

1314 For string-fields: compare lowercased values.

1315 """

1316 if not self._field: # pragma: no cover

1317 raise ValueError("Unbound Field can not be lowered!")

1318

1319 return typing.cast(Expression, self._field.lower())

1320

1321 # ... etc

1322

1323

1324 class _TypedTable:

1325 """

1326 This class is a final shared parent between TypedTable and Mixins.

1327

1328 This needs to exist because otherwise the __on_define__ of Mixins are not executed.

1329 Notably, this class exists at a level ABOVE the `metaclass=TableMeta`,

1330 because otherwise typing gets confused when Mixins are used and multiple types could satisfy

1331 generic 'T subclass of TypedTable'

1332 -> Setting 'TypedTable' as the parent for Mixin does not work at runtime (and works semi at type check time)

1333 """

1334

1335 id: "TypedField[int]"

1336

1337 _before_insert: list[BeforeInsertCallable]

1338 _after_insert: list[AfterInsertCallable]

1339 _before_update: list[BeforeUpdateCallable]

1340 _after_update: list[AfterUpdateCallable]

1341 _before_delete: list[BeforeDeleteCallable]

1342 _after_delete: list[AfterDeleteCallable]

1343

1344 @classmethod

1345 def __on_define__(cls, db: TypeDAL) -> None:

1346 """

1347 Method that can be implemented by tables to do an action after db.define is completed.

1348

1349 This can be useful if you need to add something like requires=IS_NOT_IN_DB(db, "table.field"),

1350 where you need a reference to the current database, which may not exist yet when defining the model.

1351 """

1352

1353

1354 class TypedTable(_TypedTable, metaclass=TableMeta):

1355 """

1356 Enhanded modeling system on top of pydal's Table that adds typing and additional functionality.

1357 """

1358

1359 # set up by 'new':

1360 _row: Row | None = None

1361

1362 _with: list[str]

1363

1364 def _setup_instance_methods(self) -> None:

1365 self.as_dict = self._as_dict # type: ignore

1366 self.__json__ = self.as_json = self._as_json # type: ignore

1367 # self.as_yaml = self._as_yaml # type: ignore

1368 self.as_xml = self._as_xml # type: ignore

1369

1370 self.update = self._update # type: ignore

1371

1372 self.delete_record = self._delete_record # type: ignore

1373 self.update_record = self._update_record # type: ignore

1374

1375 def __new__(

1376 cls, row_or_id: typing.Union[Row, Query, pydal.objects.Set, int, str, None, "TypedTable"] = None, **filters: Any

1377 ) -> typing.Self:

1378 """

1379 Create a Typed Rows model instance from an existing row, ID or query.

1380

1381 Examples:

1382 MyTable(1)

1383 MyTable(id=1)

1384 MyTable(MyTable.id == 1)

1385 """

1386 table = cls._ensure_table_defined()

1387 inst = super().__new__(cls)

1388

1389 if isinstance(row_or_id, TypedTable):

1390 # existing typed table instance!

1391 return typing.cast(Self, row_or_id)

1392

1393 elif isinstance(row_or_id, pydal.objects.Row):

1394 row = row_or_id

1395 elif row_or_id is not None:

1396 row = table(row_or_id, **filters)

1397 elif filters:

1398 row = table(**filters)

1399 else:

1400 # dummy object

1401 return inst

1402

1403 if not row:

1404 return None # type: ignore

1405

1406 inst._row = row

1407 inst.__dict__.update(row)

1408 inst._setup_instance_methods()

1409 return inst

1410

1411 def __iter__(self) -> typing.Generator[Any, None, None]:

1412 """

1413 Allows looping through the columns.

1414 """

1415 row = self._ensure_matching_row()

1416 yield from iter(row)

1417

1418 def __getitem__(self, item: str) -> Any:

1419 """

1420 Allows dictionary notation to get columns.

1421 """

1422 if item in self.__dict__:

1423 return self.__dict__.get(item)

1424

1425 # fallback to lookup in row

1426 if self._row:

1427 return self._row[item]

1428

1429 # nothing found!

1430 raise KeyError(item)

1431

1432 def __getattr__(self, item: str) -> Any:

1433 """

1434 Allows dot notation to get columns.

1435 """

1436 if value := self.get(item):

1437 return value

1438

1439 raise AttributeError(item)

1440

1441 def get(self, item: str, default: Any = None) -> Any:

1442 """

1443 Try to get a column from this instance, else return default.

1444 """

1445 try:

1446 return self.__getitem__(item)

1447 except KeyError:

1448 return default

1449

1450 def __setitem__(self, key: str, value: Any) -> None:

1451 """

1452 Data can both be updated via dot and dict notation.

1453 """

1454 return setattr(self, key, value)

1455

1456 def __int__(self) -> int:

1457 """

1458 Calling int on a model instance will return its id.

1459 """

1460 return getattr(self, "id", 0)

1461

1462 def __bool__(self) -> bool:

1463 """

1464 If the instance has an underlying row with data, it is truthy.

1465 """

1466 return bool(getattr(self, "_row", False))

1467

1468 def _ensure_matching_row(self) -> Row:

1469 if not getattr(self, "_row", None):

1470 raise EnvironmentError("Trying to access non-existant row. Maybe it was deleted or not yet initialized?")

1471 return self._row

1472

1473 def __repr__(self) -> str:

1474 """

1475 String representation of the model instance.

1476 """

1477 model_name = self.__class__.__name__

1478 model_data = {}

1479

1480 if self._row:

1481 model_data = self._row.as_json()

1482

1483 details = model_name

1484 details += f"({model_data})"

1485

1486 if relationships := getattr(self, "_with", []):

1487 details += f" + {relationships}"

1488

1489 return f"<{details}>"

1490

1491 # serialization

1492 # underscore variants work for class instances (set up by _setup_instance_methods)

1493

1494 @classmethod

1495 def as_dict(cls, flat: bool = False, sanitize: bool = True) -> AnyDict:

1496 """

1497 Dump the object to a plain dict.

1498

1499 Can be used as both a class or instance method:

1500 - dumps the table info if it's a class

1501 - dumps the row info if it's an instance (see _as_dict)

1502 """

1503 table = cls._ensure_table_defined()

1504 result = table.as_dict(flat, sanitize)

1505 return typing.cast(AnyDict, result)

1506

1507 @classmethod

1508 def as_json(cls, sanitize: bool = True, indent: Optional[int] = None, **kwargs: Any) -> str:

1509 """

1510 Dump the object to json.

1511

1512 Can be used as both a class or instance method:

1513 - dumps the table info if it's a class

1514 - dumps the row info if it's an instance (see _as_json)

1515 """

1516 data = cls.as_dict(sanitize=sanitize)

1517 return as_json.encode(data, indent=indent, **kwargs)

1518

1519 @classmethod

1520 def as_xml(cls, sanitize: bool = True) -> str: # pragma: no cover

1521 """

1522 Dump the object to xml.

1523

1524 Can be used as both a class or instance method:

1525 - dumps the table info if it's a class

1526 - dumps the row info if it's an instance (see _as_xml)

1527 """

1528 table = cls._ensure_table_defined()

1529 return typing.cast(str, table.as_xml(sanitize))

1530

1531 @classmethod

1532 def as_yaml(cls, sanitize: bool = True) -> str:

1533 """

1534 Dump the object to yaml.

1535

1536 Can be used as both a class or instance method:

1537 - dumps the table info if it's a class

1538 - dumps the row info if it's an instance (see _as_yaml)

1539 """

1540 table = cls._ensure_table_defined()

1541 return typing.cast(str, table.as_yaml(sanitize))

1542

1543 def _as_dict(

1544 self, datetime_to_str: bool = False, custom_types: typing.Iterable[type] | type | None = None

1545 ) -> AnyDict:

1546 row = self._ensure_matching_row()

1547

1548 result = row.as_dict(datetime_to_str=datetime_to_str, custom_types=custom_types)

1549

1550 def asdict_method(obj: Any) -> Any: # pragma: no cover

1551 if hasattr(obj, "_as_dict"): # typedal

1552 return obj._as_dict()

1553 elif hasattr(obj, "as_dict"): # pydal

1554 return obj.as_dict()

1555 else: # something else??

1556 return obj.__dict__

1557

1558 if _with := getattr(self, "_with", None):

1559 for relationship in _with:

1560 data = self.get(relationship)

1561

1562 if isinstance(data, list):

1563 data = [asdict_method(_) for _ in data]

1564 elif data:

1565 data = asdict_method(data)

1566

1567 result[relationship] = data

1568

1569 return typing.cast(AnyDict, result)

1570

1571 def _as_json(

1572 self,

1573 default: typing.Callable[[Any], Any] = None,

1574 indent: Optional[int] = None,

1575 **kwargs: Any,

1576 ) -> str:

1577 data = self._as_dict()

1578 return as_json.encode(data, default=default, indent=indent, **kwargs)

1579

1580 def _as_xml(self, sanitize: bool = True) -> str: # pragma: no cover

1581 row = self._ensure_matching_row()

1582 return typing.cast(str, row.as_xml(sanitize))

1583

1584 # def _as_yaml(self, sanitize: bool = True) -> str:

1585 # row = self._ensure_matching_row()

1586 # return typing.cast(str, row.as_yaml(sanitize))

1587

1588 def __setattr__(self, key: str, value: Any) -> None:

1589 """

1590 When setting a property on a Typed Table model instance, also update the underlying row.

1591 """

1592 if self._row and key in self._row.__dict__ and not callable(value):

1593 # enables `row.key = value; row.update_record()`

1594 self._row[key] = value

1595

1596 super().__setattr__(key, value)

1597

1598 @classmethod

1599 def update(cls: typing.Type[T_MetaInstance], query: Query, **fields: Any) -> T_MetaInstance | None:

1600 """

1601 Update one record.

1602

1603 Example:

1604 MyTable.update(MyTable.id == 1, name="NewName") -> MyTable

1605 """

1606 # todo: update multiple?

1607 if record := cls(query):

1608 return record.update_record(**fields)

1609 else:

1610 return None

1611

1612 def _update(self: T_MetaInstance, **fields: Any) -> T_MetaInstance:

1613 row = self._ensure_matching_row()

1614 row.update(**fields)

1615 self.__dict__.update(**fields)

1616 return self

1617

1618 def _update_record(self: T_MetaInstance, **fields: Any) -> T_MetaInstance:

1619 row = self._ensure_matching_row()

1620 new_row = row.update_record(**fields)

1621 self.update(**new_row)

1622 return self

1623

1624 def update_record(self: T_MetaInstance, **fields: Any) -> T_MetaInstance: # pragma: no cover

1625 """

1626 Here as a placeholder for _update_record.

1627

1628 Will be replaced on instance creation!

1629 """

1630 return self._update_record(**fields)

1631

1632 def _delete_record(self) -> int:

1633 """

1634 Actual logic in `pydal.helpers.classes.RecordDeleter`.

1635 """

1636 row = self._ensure_matching_row()

1637 result = row.delete_record()

1638 self.__dict__ = {} # empty self, since row is no more.

1639 self._row = None # just to be sure

1640 self._setup_instance_methods()

1641 # ^ instance methods might've been deleted by emptying dict,

1642 # but we still want .as_dict to show an error, not the table's as_dict.

1643 return typing.cast(int, result)

1644

1645 def delete_record(self) -> int: # pragma: no cover

1646 """

1647 Here as a placeholder for _delete_record.

1648

1649 Will be replaced on instance creation!

1650 """

1651 return self._delete_record()

1652

1653 # __del__ is also called on the end of a scope so don't remove records on every del!!

1654

1655 # pickling:

1656

1657 def __getstate__(self) -> AnyDict:

1658 """

1659 State to save when pickling.

1660

1661 Prevents db connection from being pickled.

1662 Similar to as_dict but without changing the data of the relationships (dill does that recursively)

1663 """

1664 row = self._ensure_matching_row()

1665 result: AnyDict = row.as_dict()

1666

1667 if _with := getattr(self, "_with", None):

1668 result["_with"] = _with

1669 for relationship in _with:

1670 data = self.get(relationship)

1671

1672 result[relationship] = data

1673

1674 result["_row"] = self._row.as_json() if self._row else ""

1675 return result

1676

1677 def __setstate__(self, state: AnyDict) -> None:

1678 """

1679 Used by dill when loading from a bytestring.

1680 """

1681 # as_dict also includes table info, so dump as json to only get the actual row data

1682 # then create a new (more empty) row object:

1683 state["_row"] = Row(json.loads(state["_row"]))

1684 self.__dict__ |= state

1685

1686

1687# backwards compat:

1688 TypedRow = TypedTable

1689

1690

1691 class TypedRows(typing.Collection[T_MetaInstance], Rows):

1692 """

1693 Slighly enhaned and typed functionality on top of pydal Rows (the result of a select).

1694 """

1695

1696 records: dict[int, T_MetaInstance]

1697 # _rows: Rows

1698 model: typing.Type[T_MetaInstance]

1699 metadata: Metadata

1700

1701 # pseudo-properties: actually stored in _rows

1702 db: TypeDAL

1703 colnames: list[str]

1704 fields: list[Field]

1705 colnames_fields: list[Field]

1706 response: list[tuple[Any, ...]]

1707

1708 def __init__(

1709 self,

1710 rows: Rows,

1711 model: typing.Type[T_MetaInstance],

1712 records: dict[int, T_MetaInstance] = None,

1713 metadata: Metadata = None,

1714 ) -> None:

1715 """

1716 Should not be called manually!

1717

1718 Normally, the `records` from an existing `Rows` object are used

1719 but these can be overwritten with a `records` dict.

1720 `metadata` can be any (un)structured data

1721 `model` is a Typed Table class

1722 """

1723 records = records or {row.id: model(row) for row in rows}

1724 super().__init__(rows.db, records, rows.colnames, rows.compact, rows.response, rows.fields)

1725 self.model = model

1726 self.metadata = metadata or {}

1727 self.colnames = rows.colnames

1728

1729 def __len__(self) -> int:

1730 """

1731 Return the count of rows.

1732 """

1733 return len(self.records)

1734

1735 def __iter__(self) -> typing.Iterator[T_MetaInstance]:

1736 """

1737 Loop through the rows.

1738 """

1739 yield from self.records.values()

1740

1741 def __contains__(self, ind: Any) -> bool:

1742 """

1743 Check if an id exists in this result set.

1744 """

1745 return ind in self.records

1746

1747 def first(self) -> T_MetaInstance | None:

1748 """

1749 Get the row with the lowest id.

1750 """

1751 if not self.records:

1752 return None

1753

1754 return next(iter(self))

1755

1756 def last(self) -> T_MetaInstance | None:

1757 """

1758 Get the row with the highest id.

1759 """

1760 if not self.records:

1761 return None

1762

1763 max_id = max(self.records.keys())

1764 return self[max_id]

1765

1766 def find(

1767 self, f: typing.Callable[[T_MetaInstance], Query], limitby: tuple[int, int] = None

1768 ) -> "TypedRows[T_MetaInstance]":

1769 """

1770 Returns a new Rows object, a subset of the original object, filtered by the function `f`.

1771 """

1772 if not self.records:

1773 return self.__class__(self, self.model, {})

1774

1775 records = {}

1776 if limitby:

1777 _min, _max = limitby

1778 else:

1779 _min, _max = 0, len(self)

1780 count = 0

1781 for i, row in self.records.items():

1782 if f(row):

1783 if _min <= count:

1784 records[i] = row

1785 count += 1

1786 if count == _max:

1787 break

1788

1789 return self.__class__(self, self.model, records)

1790

1791 def exclude(self, f: typing.Callable[[T_MetaInstance], Query]) -> "TypedRows[T_MetaInstance]":

1792 """

1793 Removes elements from the calling Rows object, filtered by the function `f`, \

1794 and returns a new Rows object containing the removed elements.

1795 """

1796 if not self.records:

1797 return self.__class__(self, self.model, {})

1798 removed = {}

1799 to_remove = []

1800 for i in self.records:

1801 row = self[i]

1802 if f(row):

1803 removed[i] = self.records[i]

1804 to_remove.append(i)

1805

1806 [self.records.pop(i) for i in to_remove]

1807

1808 return self.__class__(

1809 self,

1810 self.model,

1811 removed,

1812 )

1813

1814 def sort(self, f: typing.Callable[[T_MetaInstance], Any], reverse: bool = False) -> list[T_MetaInstance]:

1815 """

1816 Returns a list of sorted elements (not sorted in place).

1817 """

1818 return [r for (r, s) in sorted(zip(self.records.values(), self), key=lambda r: f(r[1]), reverse=reverse)]

1819

1820 def __str__(self) -> str:

1821 """

1822 Simple string representation.

1823 """

1824 return f"<TypedRows with {len(self)} records>"

1825

1826 def __repr__(self) -> str:

1827 """

1828 Print a table on repr().

1829 """

1830 data = self.as_dict()

1831 headers = list(next(iter(data.values())).keys())

1832 return mktable(data, headers)

1833

1834 def group_by_value(

1835 self, *fields: "str | Field | TypedField[T]", one_result: bool = False, **kwargs: Any

1836 ) -> dict[T, list[T_MetaInstance]]:

1837 """

1838 Group the rows by a specific field (which will be the dict key).

1839 """

1840 kwargs["one_result"] = one_result

1841 result = super().group_by_value(*fields, **kwargs)

1842 return typing.cast(dict[T, list[T_MetaInstance]], result)

1843

1844 def column(self, column: str = None) -> list[Any]:

1845 """

1846 Get a list of all values in a specific column.

1847

1848 Example:

1849 rows.column('name') -> ['Name 1', 'Name 2', ...]

1850 """

1851 return typing.cast(list[Any], super().column(column))

1852

1853 def as_csv(self) -> str:

1854 """

1855 Dump the data to csv.

1856 """

1857 return typing.cast(str, super().as_csv())

1858

1859 def as_dict(

1860 self,

1861 key: str = None,

1862 compact: bool = False,

1863 storage_to_dict: bool = False,

1864 datetime_to_str: bool = False,

1865 custom_types: list[type] = None,

1866 ) -> dict[int, AnyDict]:

1867 """

1868 Get the data in a dict of dicts.

1869 """

1870 if any([key, compact, storage_to_dict, datetime_to_str, custom_types]):

1871 # functionality not guaranteed

1872 return typing.cast(

1873 dict[int, AnyDict],

1874 super().as_dict(

1875 key or "id",

1876 compact,

1877 storage_to_dict,

1878 datetime_to_str,

1879 custom_types,

1880 ),

1881 )

1882

1883 return {k: v.as_dict() for k, v in self.records.items()}

1884

1885 def as_json(self, default: typing.Callable[[Any], Any] = None, indent: Optional[int] = None, **kwargs: Any) -> str:

1886 """

1887 Turn the data into a dict and then dump to JSON.

1888 """

1889 data = self.as_list()

1890

1891 return as_json.encode(data, default=default, indent=indent, **kwargs)

1892

1893 def json(self, default: typing.Callable[[Any], Any] = None, indent: Optional[int] = None, **kwargs: Any) -> str:

1894 """

1895 Turn the data into a dict and then dump to JSON.

1896 """

1897 return self.as_json(default=default, indent=indent, **kwargs)

1898

1899 def as_list(

1900 self,

1901 compact: bool = False,

1902 storage_to_dict: bool = False,

1903 datetime_to_str: bool = False,

1904 custom_types: list[type] = None,

1905 ) -> list[AnyDict]:

1906 """

1907 Get the data in a list of dicts.

1908 """

1909 if any([compact, storage_to_dict, datetime_to_str, custom_types]):

1910 return typing.cast(list[AnyDict], super().as_list(compact, storage_to_dict, datetime_to_str, custom_types))

1911

1912 return [_.as_dict() for _ in self.records.values()]

1913

1914 def __getitem__(self, item: int) -> T_MetaInstance:

1915 """

1916 You can get a specific row by ID from a typedrows by using rows[idx] notation.

1917

1918 Since pydal's implementation differs (they expect a list instead of a dict with id keys),

1919 using rows[0] will return the first row, regardless of its id.

1920 """

1921 try:

1922 return self.records[item]

1923 except KeyError as e:

1924 if item == 0 and (row := self.first()):

1925 # special case: pydal internals think Rows.records is a list, not a dict

1926 return row

1927

1928 raise e

1929

1930 def get(self, item: int) -> typing.Optional[T_MetaInstance]:

1931 """

1932 Get a row by ID, or receive None if it isn't in this result set.

1933 """

1934 return self.records.get(item)

1935

1936 def update(self, **new_values: Any) -> bool:

1937 """

1938 Update the current rows in the database with new_values.

1939 """

1940 # cast to make mypy understand .id is a TypedField and not an int!

1941 table = typing.cast(typing.Type[TypedTable], self.model._ensure_table_defined())

1942

1943 ids = set(self.column("id"))

1944 query = table.id.belongs(ids)

1945 return bool(self.db(query).update(**new_values))

1946

1947 def delete(self) -> bool:

1948 """

1949 Delete the currently selected rows from the database.

1950 """

1951 # cast to make mypy understand .id is a TypedField and not an int!

1952 table = typing.cast(typing.Type[TypedTable], self.model._ensure_table_defined())

1953

1954 ids = set(self.column("id"))

1955 query = table.id.belongs(ids)

1956 return bool(self.db(query).delete())

1957

1958 def join(

1959 self,

1960 field: "Field | TypedField[Any]",

1961 name: str = None,

1962 constraint: Query = None,

1963 fields: list[str | Field] = None,

1964 orderby: Optional[str | Field] = None,

1965 ) -> T_MetaInstance:

1966 """

1967 This can be used to JOIN with some relationships after the initial select.

1968

1969 Using the querybuilder's .join() method is prefered!

1970 """

1971 result = super().join(field, name, constraint, fields or [], orderby)

1972 return typing.cast(T_MetaInstance, result)

1973

1974 def export_to_csv_file(

1975 self,

1976 ofile: typing.TextIO,

1977 null: Any = "<NULL>",

1978 delimiter: str = ",",

1979 quotechar: str = '"',

1980 quoting: int = csv.QUOTE_MINIMAL,

1981 represent: bool = False,

1982 colnames: list[str] = None,

1983 write_colnames: bool = True,

1984 *args: Any,

1985 **kwargs: Any,

1986 ) -> None:

1987 """

1988 Shadow export_to_csv_file from Rows, but with typing.

1989

1990 See http://web2py.com/books/default/chapter/29/06/the-database-abstraction-layer?search=export_to_csv_file#Exporting-and-importing-data

1991 """

1992 super().export_to_csv_file(

1993 ofile,

1994 null,

1995 *args,

1996 delimiter=delimiter,

1997 quotechar=quotechar,

1998 quoting=quoting,

1999 represent=represent,

2000 colnames=colnames or self.colnames,

2001 write_colnames=write_colnames,

2002 **kwargs,

2003 )

2004

2005 @classmethod

2006 def from_rows(

2007 cls, rows: Rows, model: typing.Type[T_MetaInstance], metadata: Metadata = None

2008 ) -> "TypedRows[T_MetaInstance]":

2009 """

2010 Internal method to convert a Rows object to a TypedRows.

2011 """

2012 return cls(rows, model, metadata=metadata)

2013

2014 def __getstate__(self) -> AnyDict:

2015 """

2016 Used by dill to dump to bytes (exclude db connection etc).

2017 """

2018 return {

2019 "metadata": json.dumps(self.metadata, default=str),

2020 "records": self.records,

2021 "model": str(self.model._table),

2022 "colnames": self.colnames,

2023 }

2024

2025 def __setstate__(self, state: AnyDict) -> None:

2026 """

2027 Used by dill when loading from a bytestring.

2028 """

2029 state["metadata"] = json.loads(state["metadata"])

2030 self.__dict__.update(state)

2031 # db etc. set after undill by caching.py

2032

2033

2034 from .caching import ( # noqa: E402

2035 _remove_cache,

2036 _TypedalCache,

2037 _TypedalCacheDependency,

2038 create_and_hash_cache_key,

2039 get_expire,

2040 load_from_cache,

2041 save_to_cache,

2042)

2043

2044

2045 class QueryBuilder(typing.Generic[T_MetaInstance]):

2046 """

2047 Abstration on top of pydal's query system.

2048 """

2049

2050 model: typing.Type[T_MetaInstance]

2051 query: Query

2052 select_args: list[Any]

2053 select_kwargs: SelectKwargs

2054 relationships: dict[str, Relationship[Any]]

2055 metadata: Metadata

2056

2057 def __init__(

2058 self,

2059 model: typing.Type[T_MetaInstance],

2060 add_query: Optional[Query] = None,

2061 select_args: Optional[list[Any]] = None,

2062 select_kwargs: Optional[SelectKwargs] = None,

2063 relationships: dict[str, Relationship[Any]] = None,

2064 metadata: Metadata = None,

2065 ):

2066 """

2067 Normally, you wouldn't manually initialize a QueryBuilder but start using a method on a TypedTable.

2068

2069 Example:

2070 MyTable.where(...) -> QueryBuilder[MyTable]

2071 """

2072 self.model = model

2073 table = model._ensure_table_defined()

2074 default_query = typing.cast(Query, table.id > 0)

2075 self.query = add_query or default_query

2076 self.select_args = select_args or []

2077 self.select_kwargs = select_kwargs or {}

2078 self.relationships = relationships or {}

2079 self.metadata = metadata or {}

2080

2081 def __str__(self) -> str:

2082 """

2083 Simple string representation for the query builder.

2084 """

2085 return f"QueryBuilder for {self.model}"

2086

2087 def __repr__(self) -> str:

2088 """

2089 Advanced string representation for the query builder.

2090 """

2091 return (

2092 f"<QueryBuilder for {self.model} with "

2093 f"{len(self.select_args)} select args; "

2094 f"{len(self.select_kwargs)} select kwargs; "

2095 f"{len(self.relationships)} relationships; "

2096 f"query: {bool(self.query)}; "

2097 f"metadata: {self.metadata}; "

2098 f">"

2099 )

2100

2101 def __bool__(self) -> bool:

2102 """

2103 Querybuilder is truthy if it has rows.

2104 """

2105 return self.count() > 0

2106

2107 def _extend(

2108 self,

2109 add_query: Optional[Query] = None,

2110 overwrite_query: Optional[Query] = None,

2111 select_args: Optional[list[Any]] = None,

2112 select_kwargs: Optional[SelectKwargs] = None,

2113 relationships: dict[str, Relationship[Any]] = None,

2114 metadata: Metadata = None,

2115 ) -> "QueryBuilder[T_MetaInstance]":

2116 return QueryBuilder(

2117 self.model,

2118 (add_query & self.query) if add_query else overwrite_query or self.query,

2119 (self.select_args + select_args) if select_args else self.select_args,

2120 (self.select_kwargs | select_kwargs) if select_kwargs else self.select_kwargs,

2121 (self.relationships | relationships) if relationships else self.relationships,

2122 (self.metadata | (metadata or {})) if metadata else self.metadata,

2123 )

2124

2125 def select(self, *fields: Any, **options: Unpack[SelectKwargs]) -> "QueryBuilder[T_MetaInstance]":

2126 """

2127 Fields: database columns by name ('id'), by field reference (table.id) or other (e.g. table.ALL).

2128

2129 Options:

2130 paraphrased from the web2py pydal docs,

2131 For more info, see http://www.web2py.com/books/default/chapter/29/06/the-database-abstraction-layer#orderby-groupby-limitby-distinct-having-orderby_on_limitby-join-left-cache

2132

2133 orderby: field(s) to order by. Supported:

2134 table.name - sort by name, ascending

2135 ~table.name - sort by name, descending

2136 <random> - sort randomly

2137 table.name|table.id - sort by two fields (first name, then id)

2138

2139 groupby, having: together with orderby:

2140 groupby can be a field (e.g. table.name) to group records by

2141 having can be a query, only those `having` the condition are grouped

2142

2143 limitby: tuple of min and max. When using the query builder, .paginate(limit, page) is recommended.

2144 distinct: bool/field. Only select rows that differ

2145 orderby_on_limitby (bool, default: True): by default, an implicit orderby is added when doing limitby.

2146 join: othertable.on(query) - do an INNER JOIN. Using TypeDAL relationships with .join() is recommended!

2147 left: othertable.on(query) - do a LEFT JOIN. Using TypeDAL relationships with .join() is recommended!

2148 cache: cache the query result to speed up repeated queries; e.g. (cache=(cache.ram, 3600), cacheable=True)

2149 """

2150 return self._extend(select_args=list(fields), select_kwargs=options)

2151

2152 def where(

2153 self,

2154 *queries_or_lambdas: Query | typing.Callable[[typing.Type[T_MetaInstance]], Query],

2155 **filters: Any,

2156 ) -> "QueryBuilder[T_MetaInstance]":

2157 """

2158 Extend the builder's query.

2159

2160 Can be used in multiple ways:

2161 .where(Query) -> with a direct query such as `Table.id == 5`

2162 .where(lambda table: table.id == 5) -> with a query via a lambda

2163 .where(id=5) -> via keyword arguments

2164

2165 When using multiple where's, they will be ANDed:

2166 .where(lambda table: table.id == 5).where(lambda table: table.id == 6) == (table.id == 5) & (table.id=6)

2167 When passing multiple queries to a single .where, they will be ORed:

2168 .where(lambda table: table.id == 5, lambda table: table.id == 6) == (table.id == 5) | (table.id=6)

2169 """

2170 new_query = self.query

2171 table = self.model._ensure_table_defined()

2172

2173 for field, value in filters.items():

2174 new_query &= table[field] == value

2175

2176 subquery: DummyQuery | Query = DummyQuery()

2177 for query_or_lambda in queries_or_lambdas:

2178 if isinstance(query_or_lambda, _Query):

2179 subquery |= typing.cast(Query, query_or_lambda)

2180 elif callable(query_or_lambda):

2181 if result := query_or_lambda(self.model):

2182 subquery |= result

2183 elif isinstance(query_or_lambda, (Field, _Field)) or is_typed_field(query_or_lambda):

2184 subquery |= typing.cast(Query, query_or_lambda != None)

2185 else:

2186 raise ValueError(f"Unexpected query type ({type(query_or_lambda)}).")

2187

2188 if subquery:

2189 new_query &= subquery

2190

2191 return self._extend(overwrite_query=new_query)

2192

2193 def join(

2194 self,

2195 *fields: str | typing.Type[TypedTable],

2196 method: JOIN_OPTIONS = None,

2197 on: OnQuery | list[Expression] | Expression = None,

2198 condition: Condition = None,

2199 ) -> "QueryBuilder[T_MetaInstance]":

2200 """

2201 Include relationship fields in the result.

2202

2203 `fields` can be names of Relationships on the current model.

2204 If no fields are passed, all will be used.

2205

2206 By default, the `method` defined in the relationship is used.

2207 This can be overwritten with the `method` keyword argument (left or inner)

2208 """

2209 # todo: allow limiting amount of related rows returned for join?

2210

2211 relationships = self.model.get_relationships()

2212

2213 if condition and on:

2214 raise ValueError("condition and on can not be used together!")

2215 elif condition:

2216 if len(fields) != 1:

2217 raise ValueError("join(field, condition=...) can only be used with exactly one field!")

2218

2219 if isinstance(condition, pydal.objects.Query):

2220 condition = as_lambda(condition)

2221

2222 relationships = {str(fields[0]): relationship(fields[0], condition=condition, join=method)}

2223 elif on:

2224 if len(fields) != 1:

2225 raise ValueError("join(field, on=...) can only be used with exactly one field!")

2226

2227 if isinstance(on, pydal.objects.Expression):

2228 on = [on]

2229

2230 if isinstance(on, list):

2231 on = as_lambda(on)

2232 relationships = {str(fields[0]): relationship(fields[0], on=on, join=method)}

2233

2234 else:

2235 if fields:

2236 # join on every relationship

2237 relationships = {str(k): relationships[str(k)] for k in fields}

2238

2239 if method:

2240 relationships = {str(k): r.clone(join=method) for k, r in relationships.items()}

2241

2242 return self._extend(relationships=relationships)

2243

2244 def cache(

2245 self, *deps: Any, expires_at: Optional[dt.datetime] = None, ttl: Optional[int | dt.timedelta] = None

2246 ) -> "QueryBuilder[T_MetaInstance]":

2247 """

2248 Enable caching for this query to load repeated calls from a dill row \

2249 instead of executing the sql and collecing matching rows again.

2250 """

2251 existing = self.metadata.get("cache", {})

2252

2253 metadata: Metadata = {}

2254

2255 cache_meta = typing.cast(

2256 CacheMetadata,

2257 self.metadata.get("cache", {})

2258 | {

2259 "enabled": True,

2260 "depends_on": existing.get("depends_on", []) + [str(_) for _ in deps],

2261 "expires_at": get_expire(expires_at=expires_at, ttl=ttl),

2262 },

2263 )

2264

2265 metadata["cache"] = cache_meta

2266 return self._extend(metadata=metadata)

2267

2268 def _get_db(self) -> TypeDAL:

2269 if db := self.model._db:

2270 return db

2271 else: # pragma: no cover

2272 raise EnvironmentError("@define or db.define is not called on this class yet!")

2273

2274 def _select_arg_convert(self, arg: Any) -> Any:

2275 # typedfield are not really used at runtime anymore, but leave it in for safety:

2276 if isinstance(arg, TypedField): # pragma: no cover

2277 arg = arg._field

2278

2279 return arg

2280

2281 def delete(self) -> list[int]:

2282 """

2283 Based on the current query, delete rows and return a list of deleted IDs.

2284 """

2285 db = self._get_db()

2286 removed_ids = [_.id for _ in db(self.query).select("id")]

2287 if db(self.query).delete():

2288 # success!

2289 return removed_ids

2290

2291 return []

2292

2293 def _delete(self) -> str:

2294 db = self._get_db()

2295 return str(db(self.query)._delete())

2296

2297 def update(self, **fields: Any) -> list[int]:

2298 """

2299 Based on the current query, update `fields` and return a list of updated IDs.

2300 """

2301 # todo: limit?

2302 db = self._get_db()

2303 updated_ids = db(self.query).select("id").column("id")

2304 if db(self.query).update(**fields):

2305 # success!

2306 return updated_ids

2307

2308 return []

2309

2310 def _update(self, **fields: Any) -> str:

2311 db = self._get_db()

2312 return str(db(self.query)._update(**fields))

2313

2314 def _before_query(self, mut_metadata: Metadata, add_id: bool = True) -> tuple[Query, list[Any], SelectKwargs]:

2315 select_args = [self._select_arg_convert(_) for _ in self.select_args] or [self.model.ALL]

2316 select_kwargs = self.select_kwargs.copy()

2317 query = self.query

2318 model = self.model

2319 mut_metadata["query"] = query

2320 # require at least id of main table:

2321 select_fields = ", ".join([str(_) for _ in select_args])

2322 tablename = str(model)

2323

2324 if add_id and f"{tablename}.id" not in select_fields:

2325 # fields of other selected, but required ID is missing.

2326 select_args.append(model.id)

2327

2328 if self.relationships:

2329 query, select_args = self._handle_relationships_pre_select(query, select_args, select_kwargs, mut_metadata)

2330

2331 return query, select_args, select_kwargs

2332

2333 def to_sql(self, add_id: bool = False) -> str:

2334 """

2335 Generate the SQL for the built query.

2336 """

2337 db = self._get_db()

2338

2339 query, select_args, select_kwargs = self._before_query({}, add_id=add_id)

2340

2341 return str(db(query)._select(*select_args, **select_kwargs))

2342

2343 def _collect(self) -> str:

2344 """

2345 Alias for to_sql, pydal-like syntax.

2346 """

2347 return self.to_sql()

2348

2349 def _collect_cached(self, metadata: Metadata) -> "TypedRows[T_MetaInstance] | None":

2350 expires_at = metadata["cache"].get("expires_at")

2351 metadata["cache"] |= {

2352 # key is partly dependant on cache metadata but not these:

2353 "key": None,

2354 "status": None,

2355 "cached_at": None,

2356 "expires_at": None,

2357 }

2358

2359 _, key = create_and_hash_cache_key(

2360 self.model,

2361 metadata,

2362 self.query,

2363 self.select_args,

2364 self.select_kwargs,

2365 self.relationships.keys(),

2366 )

2367

2368 # re-set after creating key:

2369 metadata["cache"]["expires_at"] = expires_at

2370 metadata["cache"]["key"] = key

2371

2372 return load_from_cache(key, self._get_db())

2373

2374 def execute(self, add_id: bool = False) -> Rows:

2375 """

2376 Raw version of .collect which only executes the SQL, without performing any magic afterwards.

2377 """

2378 db = self._get_db()

2379 metadata = typing.cast(Metadata, self.metadata.copy())

2380

2381 query, select_args, select_kwargs = self._before_query(metadata, add_id=add_id)

2382

2383 return db(query).select(*select_args, **select_kwargs)

2384

2385 def collect(

2386 self, verbose: bool = False, _to: typing.Type["TypedRows[Any]"] = None, add_id: bool = True

2387 ) -> "TypedRows[T_MetaInstance]":

2388 """

2389 Execute the built query and turn it into model instances, while handling relationships.

2390 """

2391 if _to is None:

2392 _to = TypedRows

2393

2394 db = self._get_db()

2395 metadata = typing.cast(Metadata, self.metadata.copy())

2396

2397 if metadata.get("cache", {}).get("enabled") and (result := self._collect_cached(metadata)):

2398 return result

2399

2400 query, select_args, select_kwargs = self._before_query(metadata, add_id=add_id)

2401

2402 metadata["sql"] = db(query)._select(*select_args, **select_kwargs)

2403

2404 if verbose: # pragma: no cover

2405 print(metadata["sql"])

2406

2407 rows: Rows = db(query).select(*select_args, **select_kwargs)

2408

2409 metadata["final_query"] = str(query)

2410 metadata["final_args"] = [str(_) for _ in select_args]

2411 metadata["final_kwargs"] = select_kwargs

2412

2413 if verbose: # pragma: no cover

2414 print(rows)

2415

2416 if not self.relationships:

2417 # easy

2418 typed_rows = _to.from_rows(rows, self.model, metadata=metadata)

2419

2420 else:

2421 # harder: try to match rows to the belonging objects

2422 # assume structure of {'table': <data>} per row.

2423 # if that's not the case, return default behavior again

2424 typed_rows = self._collect_with_relationships(rows, metadata=metadata, _to=_to)

2425

2426 # only saves if requested in metadata:

2427 return save_to_cache(typed_rows, rows)

2428

2429 def _handle_relationships_pre_select(

2430 self,

2431 query: Query,

2432 select_args: list[Any],

2433 select_kwargs: SelectKwargs,

2434 metadata: Metadata,

2435 ) -> tuple[Query, list[Any]]:

2436 db = self._get_db()

2437 model = self.model

2438

2439 metadata["relationships"] = set(self.relationships.keys())

2440

2441 # query = self._update_query_for_inner(db, model, query)

2442 join = []

2443 for key, relation in self.relationships.items():

2444 if not relation.condition or relation.join != "inner":

2445 continue

2446

2447 other = relation.get_table(db)

2448 other = other.with_alias(f"{key}_{hash(relation)}")

2449 join.append(other.on(relation.condition(model, other)))

2450

2451 if limitby := select_kwargs.pop("limitby", ()):

2452

2453 # if limitby + relationships:

2454 # 1. get IDs of main table entries that match 'query'

2455 # 2. change query to .belongs(id)

2456 # 3. add joins etc

2457

2458 kwargs: SelectKwargs = select_kwargs | {"limitby": limitby}

2459 # if orderby := select_kwargs.get("orderby"):

2460 # kwargs["orderby"] = orderby

2461

2462 if join:

2463 kwargs["join"] = join

2464

2465 ids = db(query)._select(model.id, **kwargs)

2466 query = model.id.belongs(ids)

2467 metadata["ids"] = ids

2468

2469 if join:

2470 select_kwargs["join"] = join

2471

2472 left = []

2473

2474 for key, relation in self.relationships.items():

2475 other = relation.get_table(db)

2476 method: JOIN_OPTIONS = relation.join or DEFAULT_JOIN_OPTION

2477

2478 select_fields = ", ".join([str(_) for _ in select_args])

2479 pre_alias = str(other)

2480

2481 if f"{other}." not in select_fields:

2482 # no fields of other selected. add .ALL:

2483 select_args.append(other.ALL)

2484 elif f"{other}.id" not in select_fields:

2485 # fields of other selected, but required ID is missing.

2486 select_args.append(other.id)

2487

2488 if relation.on:

2489 # if it has a .on, it's always a left join!

2490 on = relation.on(model, other)

2491 if not isinstance(on, list): # pragma: no cover

2492 on = [on]

2493

2494 left.extend(on)

2495 elif method == "left":

2496 # .on not given, generate it:

2497 other = other.with_alias(f"{key}_{hash(relation)}")

2498 condition = typing.cast(Query, relation.condition(model, other))

2499 left.append(other.on(condition))

2500 else:

2501 # else: inner join (handled earlier)

2502 other = other.with_alias(f"{key}_{hash(relation)}") # only for replace

2503 # other = other.with_alias(f"{key}_{hash(relation)}")

2504 # query &= relation.condition(model, other)

2505

2506 # if no fields of 'other' are included, add other.ALL

2507 # else: only add other.id if missing

2508 select_fields = ", ".join([str(_) for _ in select_args])

2509

2510 post_alias = str(other).split(" AS ")[-1]

2511 if pre_alias != post_alias:

2512 # replace .select's with aliased:

2513 select_fields = select_fields.replace(

2514 f"{pre_alias}.",

2515 f"{post_alias}.",

2516 )

2517

2518 select_args = select_fields.split(", ")

2519

2520 select_kwargs["left"] = left

2521 return query, select_args

2522

2523 def _collect_with_relationships(

2524 self, rows: Rows, metadata: Metadata, _to: typing.Type["TypedRows[Any]"]

2525 ) -> "TypedRows[T_MetaInstance]":

2526 """

2527 Transform the raw rows into Typed Table model instances.

2528 """

2529 db = self._get_db()

2530 main_table = self.model._ensure_table_defined()

2531

2532 records = {}

2533 seen_relations: dict[str, set[str]] = defaultdict(set) # main id -> set of col + id for relation

2534

2535 for row in rows:

2536 main = row[main_table]

2537 main_id = main.id

2538

2539 if main_id not in records:

2540 records[main_id] = self.model(main)

2541 records[main_id]._with = list(self.relationships.keys())

2542

2543 # setup up all relationship defaults (once)

2544 for col, relationship in self.relationships.items():

2545 records[main_id][col] = [] if relationship.multiple else None

2546

2547 # now add other relationship data

2548 for column, relation in self.relationships.items():

2549 relationship_column = f"{column}_{hash(relation)}"

2550

2551 # relationship_column works for aliases with the same target column.

2552 # if col + relationship not in the row, just use the regular name.

2553

2554 relation_data = (

2555 row[relationship_column] if relationship_column in row else row[relation.get_table_name()]

2556 )

2557

2558 if relation_data.id is None:

2559 # always skip None ids

2560 continue

2561

2562 if f"{column}-{relation_data.id}" in seen_relations[main_id]:

2563 # speed up duplicates

2564 continue

2565 else:

2566 seen_relations[main_id].add(f"{column}-{relation_data.id}")

2567

2568 relation_table = relation.get_table(db)

2569 # hopefully an instance of a typed table and a regular row otherwise:

2570 instance = relation_table(relation_data) if looks_like(relation_table, TypedTable) else relation_data

2571

2572 if relation.multiple:

2573 # create list of T

2574 if not isinstance(records[main_id].get(column), list): # pragma: no cover

2575 # should already be set up before!

2576 setattr(records[main_id], column, [])

2577

2578 records[main_id][column].append(instance)

2579 else:

2580 # create single T

2581 records[main_id][column] = instance

2582

2583 return _to(rows, self.model, records, metadata=metadata)

2584

2585 def collect_or_fail(self, exception: Exception = None) -> "TypedRows[T_MetaInstance]":

2586 """

2587 Call .collect() and raise an error if nothing found.

2588

2589 Basically unwraps Optional type.

2590 """

2591 if result := self.collect():

2592 return result

2593

2594 if not exception:

2595 exception = ValueError("Nothing found!")

2596

2597 raise exception

2598

2599 def __iter__(self) -> typing.Generator[T_MetaInstance, None, None]:

2600 """

2601 You can start iterating a Query Builder object before calling collect, for ease of use.

2602 """

2603 yield from self.collect()

2604

2605 def count(self) -> int:

2606 """

2607 Return the amount of rows matching the current query.

2608 """

2609 db = self._get_db()

2610 model = self.model

2611 query = self.query

2612

2613 for key, relation in self.relationships.items():

2614 if not relation.condition or relation.join != "inner":

2615 continue

2616

2617 other = relation.get_table(db)

2618 other = other.with_alias(f"{key}_{hash(relation)}")

2619 query &= relation.condition(model, other)

2620

2621 return db(query).count()

2622

2623 def __paginate(

2624 self,

2625 limit: int,

2626 page: int = 1,

2627 ) -> "QueryBuilder[T_MetaInstance]":

2628 _from = limit * (page - 1)

2629 _to = limit * page

2630

2631 available = self.count()

2632

2633 metadata: Metadata = {}

2634

2635 metadata["pagination"] = {

2636 "limit": limit,

2637 "current_page": page,

2638 "max_page": math.ceil(available / limit),

2639 "rows": available,

2640 "min_max": (_from, _to),

2641 }

2642

2643 return self._extend(select_kwargs={"limitby": (_from, _to)}, metadata=metadata)

2644

2645 def paginate(self, limit: int, page: int = 1, verbose: bool = False) -> "PaginatedRows[T_MetaInstance]":

2646 """

2647 Paginate transforms the more readable `page` and `limit` to pydals internal limit and offset.

2648

2649 Note: when using relationships, this limit is only applied to the 'main' table and any number of extra rows \

2650 can be loaded with relationship data!

2651 """

2652 builder = self.__paginate(limit, page)

2653

2654 rows = typing.cast(PaginatedRows[T_MetaInstance], builder.collect(verbose=verbose, _to=PaginatedRows))

2655

2656 rows._query_builder = builder

2657 return rows

2658

2659 def _paginate(

2660 self,

2661 limit: int,

2662 page: int = 1,

2663 ) -> str:

2664 builder = self.__paginate(limit, page)

2665 return builder._collect()

2666

2667 def chunk(self, chunk_size: int) -> typing.Generator["TypedRows[T_MetaInstance]", Any, None]:

2668 """

2669 Generator that yields rows from a paginated source in chunks.

2670

2671 This function retrieves rows from a paginated data source in chunks of the

2672 specified `chunk_size` and yields them as TypedRows.

2673

2674 Example:

2675 ```

2676 for chunk_of_rows in Table.where(SomeTable.id > 5).chunk(100):

2677 for row in chunk_of_rows:

2678 # Process each row within the chunk.

2679 pass

2680 ```

2681 """

2682 page = 1

2683

2684 while rows := self.__paginate(chunk_size, page).collect():

2685 yield rows

2686 page += 1

2687

2688 def first(self, verbose: bool = False) -> T_MetaInstance | None:

2689 """

2690 Get the first row matching the currently built query.

2691

2692 Also adds paginate, since it would be a waste to select more rows than needed.

2693 """

2694 if row := self.paginate(page=1, limit=1, verbose=verbose).first():

2695 return self.model.from_row(row)

2696 else:

2697 return None

2698

2699 def _first(self) -> str:

2700 return self._paginate(page=1, limit=1)

2701

2702 def first_or_fail(self, exception: Exception = None, verbose: bool = False) -> T_MetaInstance:

2703 """

2704 Call .first() and raise an error if nothing found.

2705

2706 Basically unwraps Optional type.

2707 """

2708 if inst := self.first(verbose=verbose):

2709 return inst

2710

2711 if not exception:

2712 exception = ValueError("Nothing found!")

2713

2714 raise exception

2715

2716

2717 S = typing.TypeVar("S")

2718

2719

2720 class PaginatedRows(TypedRows[T_MetaInstance]):

2721 """

2722 Extension on top of rows that is used when calling .paginate() instead of .collect().

2723 """

2724

2725 _query_builder: QueryBuilder[T_MetaInstance]

2726

2727 @property

2728 def data(self) -> list[T_MetaInstance]:

2729 """

2730 Get the underlying data.

2731 """

2732 return list(self.records.values())

2733

2734 @property

2735 def pagination(self) -> Pagination:

2736 """

2737 Get all page info.

2738 """

2739 pagination_data = self.metadata["pagination"]

2740

2741 has_next_page = pagination_data["current_page"] < pagination_data["max_page"]

2742 has_prev_page = pagination_data["current_page"] > 1

2743 return {

2744 "total_items": pagination_data["rows"],

2745 "current_page": pagination_data["current_page"],

2746 "per_page": pagination_data["limit"],

2747 "total_pages": pagination_data["max_page"],

2748 "has_next_page": has_next_page,

2749 "has_prev_page": has_prev_page,

2750 "next_page": pagination_data["current_page"] + 1 if has_next_page else None,

2751 "prev_page": pagination_data["current_page"] - 1 if has_prev_page else None,

2752 }

2753

2754 def next(self) -> Self:

2755 """

2756 Get the next page.

2757 """

2758 data = self.metadata["pagination"]

2759 if data["current_page"] >= data["max_page"]:

2760 raise StopIteration("Final Page")

2761

2762 return self._query_builder.paginate(limit=data["limit"], page=data["current_page"] + 1)

2763

2764 def previous(self) -> Self:

2765 """

2766 Get the previous page.

2767 """

2768 data = self.metadata["pagination"]

2769 if data["current_page"] <= 1:

2770 raise StopIteration("First Page")

2771

2772 return self._query_builder.paginate(limit=data["limit"], page=data["current_page"] - 1)

2773

2774 def as_dict(self, *_: Any, **__: Any) -> PaginateDict: # type: ignore

2775 """

2776 Convert to a dictionary with pagination info and original data.

2777

2778 All arguments are ignored!

2779 """

2780 return {"data": super().as_dict(), "pagination": self.pagination}

2781

2782

2783 class TypedSet(pydal.objects.Set): # type: ignore # pragma: no cover

2784 """

2785 Used to make pydal Set more typed.

2786

2787 This class is not actually used, only 'cast' by TypeDAL.__call__

2788 """

2789

2790 def count(self, distinct: bool = None, cache: AnyDict = None) -> int:

2791 """

2792 Count returns an int.

2793 """

2794 result = super().count(distinct, cache)

2795 return typing.cast(int, result)

2796

2797 def select(self, *fields: Any, **attributes: Any) -> TypedRows[T_MetaInstance]:

2798 """

2799 Select returns a TypedRows of a user defined table.

2800

2801 Example:

2802 result: TypedRows[MyTable] = db(MyTable.id > 0).select()

2803

2804 for row in result:

2805 typing.reveal_type(row) # MyTable

2806 """

2807 rows = super().select(*fields, **attributes)

2808 return typing.cast(TypedRows[T_MetaInstance], rows)