Coverage for /Users/davegaeddert/Developer/dropseed/plain/plain-models/plain/models/backends/sqlite3/schema.py: 46%

1 import copy

2 from decimal import Decimal

3

4 from plain.models.backends.base.schema import BaseDatabaseSchemaEditor

5 from plain.models.backends.ddl_references import Statement

6 from plain.models.backends.utils import strip_quotes

7 from plain.models.constraints import UniqueConstraint

8 from plain.models.db import NotSupportedError

9 from plain.models.transaction import atomic

10 from plain.packages.registry import Packages

11

12

13 class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):

14 sql_delete_table = "DROP TABLE %(table)s"

15 sql_create_fk = None

16 sql_create_inline_fk = (

17 "REFERENCES %(to_table)s (%(to_column)s) DEFERRABLE INITIALLY DEFERRED"

18 )

19 sql_create_column_inline_fk = sql_create_inline_fk

20 sql_delete_column = "ALTER TABLE %(table)s DROP COLUMN %(column)s"

21 sql_create_unique = "CREATE UNIQUE INDEX %(name)s ON %(table)s (%(columns)s)"

22 sql_delete_unique = "DROP INDEX %(name)s"

23

24 def __enter__(self):

25 # Some SQLite schema alterations need foreign key constraints to be

26 # disabled. Enforce it here for the duration of the schema edition.

27 if not self.connection.disable_constraint_checking():

28 raise NotSupportedError(

29 "SQLite schema editor cannot be used while foreign key "

30 "constraint checks are enabled. Make sure to disable them "

31 "before entering a transaction.atomic() context because "

32 "SQLite does not support disabling them in the middle of "

33 "a multi-statement transaction."

34 )

35 return super().__enter__()

36

37 def __exit__(self, exc_type, exc_value, traceback):

38 self.connection.check_constraints()

39 super().__exit__(exc_type, exc_value, traceback)

40 self.connection.enable_constraint_checking()

41

42 def quote_value(self, value):

43 # The backend "mostly works" without this function and there are use

44 # cases for compiling Python without the sqlite3 libraries (e.g.

45 # security hardening).

46 try:

47 import sqlite3

48

49 value = sqlite3.adapt(value)

50 except ImportError:

51 pass

52 except sqlite3.ProgrammingError:

53 pass

54 # Manual emulation of SQLite parameter quoting

55 if isinstance(value, bool):

56 return str(int(value))

57 elif isinstance(value, Decimal | float | int):

58 return str(value)

59 elif isinstance(value, str):

60 return "'{}'".format(value.replace("'", "''"))

61 elif value is None:

62 return "NULL"

63 elif isinstance(value, bytes | bytearray | memoryview):

64 # Bytes are only allowed for BLOB fields, encoded as string

65 # literals containing hexadecimal data and preceded by a single "X"

66 # character.

67 return f"X'{value.hex()}'"

68 else:

69 raise ValueError(

70 f"Cannot quote parameter value {value! r} of type {type(value)}"

71 )

72

73 def prepare_default(self, value):

74 return self.quote_value(value)

75

76 def _is_referenced_by_fk_constraint(

77 self, table_name, column_name=None, ignore_self=False

78 ):

79 """

80 Return whether or not the provided table name is referenced by another

81 one. If `column_name` is specified, only references pointing to that

82 column are considered. If `ignore_self` is True, self-referential

83 constraints are ignored.

84 """

85 with self.connection.cursor() as cursor:

86 for other_table in self.connection.introspection.get_table_list(cursor):

87 if ignore_self and other_table.name == table_name:

88 continue

89 relations = self.connection.introspection.get_relations(

90 cursor, other_table.name

91 )

92 for constraint_column, constraint_table in relations.values():

93 if constraint_table == table_name and (

94 column_name is None or constraint_column == column_name

95 ):

96 return True

97 return False

98

99 def alter_db_table(

100 self, model, old_db_table, new_db_table, disable_constraints=True

101 ):

102 if (

103 not self.connection.features.supports_atomic_references_rename

104 and disable_constraints

105 and self._is_referenced_by_fk_constraint(old_db_table)

106 ):

107 if self.connection.in_atomic_block:

108 raise NotSupportedError(

109 f"Renaming the {old_db_table! r} table while in a transaction is not "

110 "supported on SQLite < 3.26 because it would break referential "

111 "integrity. Try adding `atomic = False` to the Migration class."

112 )

113 self.connection.enable_constraint_checking()

114 super().alter_db_table(model, old_db_table, new_db_table)

115 self.connection.disable_constraint_checking()

116 else:

117 super().alter_db_table(model, old_db_table, new_db_table)

118

119 def alter_field(self, model, old_field, new_field, strict=False):

120 if not self._field_should_be_altered(old_field, new_field):

121 return

122 old_field_name = old_field.name

123 table_name = model._meta.db_table

124 _, old_column_name = old_field.get_attname_column()

125 if (

126 new_field.name != old_field_name

127 and not self.connection.features.supports_atomic_references_rename

128 and self._is_referenced_by_fk_constraint(

129 table_name, old_column_name, ignore_self=True

130 )

131 ):

132 if self.connection.in_atomic_block:

133 raise NotSupportedError(

134 f"Renaming the {model._meta.db_table! r}.{old_field_name! r} column while in a transaction is not "

135 "supported on SQLite < 3.26 because it would break referential "

136 "integrity. Try adding `atomic = False` to the Migration class."

137 )

138 with atomic(self.connection.alias):

139 super().alter_field(model, old_field, new_field, strict=strict)

140 # Follow SQLite's documented procedure for performing changes

141 # that don't affect the on-disk content.

142 # https://sqlite.org/lang_altertable.html#otheralter

143 with self.connection.cursor() as cursor:

144 schema_version = cursor.execute("PRAGMA schema_version").fetchone()[

145 0

146 ]

147 cursor.execute("PRAGMA writable_schema = 1")

148 references_template = f' REFERENCES "{table_name}" ("%s") '

149 new_column_name = new_field.get_attname_column()[1]

150 search = references_template % old_column_name

151 replacement = references_template % new_column_name

152 cursor.execute(

153 "UPDATE sqlite_master SET sql = replace(sql, %s, %s)",

154 (search, replacement),

155 )

156 cursor.execute("PRAGMA schema_version = %d" % (schema_version + 1))

157 cursor.execute("PRAGMA writable_schema = 0")

158 # The integrity check will raise an exception and rollback

159 # the transaction if the sqlite_master updates corrupt the

160 # database.

161 cursor.execute("PRAGMA integrity_check")

162 # Perform a VACUUM to refresh the database representation from

163 # the sqlite_master table.

164 with self.connection.cursor() as cursor:

165 cursor.execute("VACUUM")

166 else:

167 super().alter_field(model, old_field, new_field, strict=strict)

168

169 def _remake_table(

170 self, model, create_field=None, delete_field=None, alter_fields=None

171 ):

172 """

173 Shortcut to transform a model from old_model into new_model

174

175 This follows the correct procedure to perform non-rename or column

176 addition operations based on SQLite's documentation

177

178 https://www.sqlite.org/lang_altertable.html#caution

179

180 The essential steps are:

181 1. Create a table with the updated definition called "new__app_model"

182 2. Copy the data from the existing "app_model" table to the new table

183 3. Drop the "app_model" table

184 4. Rename the "new__app_model" table to "app_model"

185 5. Restore any index of the previous "app_model" table.

186 """

187

188 # Self-referential fields must be recreated rather than copied from

189 # the old model to ensure their remote_field.field_name doesn't refer

190 # to an altered field.

191 def is_self_referential(f):

192 return f.is_relation and f.remote_field.model is model

193

194 # Work out the new fields dict / mapping

195 body = {

196 f.name: f.clone() if is_self_referential(f) else f

197 for f in model._meta.local_concrete_fields

198 }

199 # Since mapping might mix column names and default values,

200 # its values must be already quoted.

201 mapping = {

202 f.column: self.quote_name(f.column)

203 for f in model._meta.local_concrete_fields

204 }

205 # If any of the new or altered fields is introducing a new PK,

206 # remove the old one

207 restore_pk_field = None

208 alter_fields = alter_fields or []

209 if getattr(create_field, "primary_key", False) or any(

210 getattr(new_field, "primary_key", False) for _, new_field in alter_fields

211 ):

212 for name, field in list(body.items()):

213 if field.primary_key and not any(

214 # Do not remove the old primary key when an altered field

215 # that introduces a primary key is the same field.

216 name == new_field.name

217 for _, new_field in alter_fields

218 ):

219 field.primary_key = False

220 restore_pk_field = field

221 if field.auto_created:

222 del body[name]

223 del mapping[field.column]

224 # Add in any created fields

225 if create_field:

226 body[create_field.name] = create_field

227 # Choose a default and insert it into the copy map

228 if not create_field.many_to_many and create_field.concrete:

229 mapping[create_field.column] = self.prepare_default(

230 self.effective_default(create_field),

231 )

232 # Add in any altered fields

233 for alter_field in alter_fields:

234 old_field, new_field = alter_field

235 body.pop(old_field.name, None)

236 mapping.pop(old_field.column, None)

237 body[new_field.name] = new_field

238 if old_field.null and not new_field.null:

239 case_sql = f"coalesce({self.quote_name(old_field.column)}, {self.prepare_default(self.effective_default(new_field))})"

240 mapping[new_field.column] = case_sql

241 else:

242 mapping[new_field.column] = self.quote_name(old_field.column)

243 # Remove any deleted fields

244 if delete_field:

245 del body[delete_field.name]

246 del mapping[delete_field.column]

247 # Remove any implicit M2M tables

248 if (

249 delete_field.many_to_many

250 and delete_field.remote_field.through._meta.auto_created

251 ):

252 return self.delete_model(delete_field.remote_field.through)

253 # Work inside a new app registry

254 packages = Packages()

255

256 indexes = model._meta.indexes

257 if delete_field:

258 indexes = [

259 index for index in indexes if delete_field.name not in index.fields

260 ]

261

262 constraints = list(model._meta.constraints)

263

264 # Provide isolated instances of the fields to the new model body so

265 # that the existing model's internals aren't interfered with when

266 # the dummy model is constructed.

267 body_copy = copy.deepcopy(body)

268

269 # Construct a new model with the new fields to allow self referential

270 # primary key to resolve to. This model won't ever be materialized as a

271 # table and solely exists for foreign key reference resolution purposes.

272 # This wouldn't be required if the schema editor was operating on model

273 # states instead of rendered models.

274 meta_contents = {

275 "package_label": model._meta.package_label,

276 "db_table": model._meta.db_table,

277 "indexes": indexes,

278 "constraints": constraints,

279 "packages": packages,

280 }

281 meta = type("Meta", (), meta_contents)

282 body_copy["Meta"] = meta

283 body_copy["__module__"] = model.__module__

284 type(model._meta.object_name, model.__bases__, body_copy)

285

286 # Construct a model with a renamed table name.

287 body_copy = copy.deepcopy(body)

288 meta_contents = {

289 "package_label": model._meta.package_label,

290 "db_table": f"new__{strip_quotes(model._meta.db_table)}",

291 "indexes": indexes,

292 "constraints": constraints,

293 "packages": packages,

294 }

295 meta = type("Meta", (), meta_contents)

296 body_copy["Meta"] = meta

297 body_copy["__module__"] = model.__module__

298 new_model = type(f"New{model._meta.object_name}", model.__bases__, body_copy)

299

300 # Create a new table with the updated schema.

301 self.create_model(new_model)

302

303 # Copy data from the old table into the new table

304 self.execute(

305 "INSERT INTO {} ({}) SELECT {} FROM {}".format(

306 self.quote_name(new_model._meta.db_table),

307 ", ".join(self.quote_name(x) for x in mapping),

308 ", ".join(mapping.values()),

309 self.quote_name(model._meta.db_table),

310 )

311 )

312

313 # Delete the old table to make way for the new

314 self.delete_model(model, handle_autom2m=False)

315

316 # Rename the new table to take way for the old

317 self.alter_db_table(

318 new_model,

319 new_model._meta.db_table,

320 model._meta.db_table,

321 disable_constraints=False,

322 )

323

324 # Run deferred SQL on correct table

325 for sql in self.deferred_sql:

326 self.execute(sql)

327 self.deferred_sql = []

328 # Fix any PK-removed field

329 if restore_pk_field:

330 restore_pk_field.primary_key = True

331

332 def delete_model(self, model, handle_autom2m=True):

333 if handle_autom2m:

334 super().delete_model(model)

335 else:

336 # Delete the table (and only that)

337 self.execute(

338 self.sql_delete_table

339 % {

340 "table": self.quote_name(model._meta.db_table),

341 }

342 )

343 # Remove all deferred statements referencing the deleted table.

344 for sql in list(self.deferred_sql):

345 if isinstance(sql, Statement) and sql.references_table(

346 model._meta.db_table

347 ):

348 self.deferred_sql.remove(sql)

349

350 def add_field(self, model, field):

351 """Create a field on a model."""

352 # Special-case implicit M2M tables.

353 if field.many_to_many and field.remote_field.through._meta.auto_created:

354 self.create_model(field.remote_field.through)

355 elif (

356 # Primary keys and unique fields are not supported in ALTER TABLE

357 # ADD COLUMN.

358 field.primary_key

359 or field.unique

360 or

361 # Fields with default values cannot by handled by ALTER TABLE ADD

362 # COLUMN statement because DROP DEFAULT is not supported in

363 # ALTER TABLE.

364 not field.null

365 or self.effective_default(field) is not None

366 ):

367 self._remake_table(model, create_field=field)

368 else:

369 super().add_field(model, field)

370

371 def remove_field(self, model, field):

372 """

373 Remove a field from a model. Usually involves deleting a column,

374 but for M2Ms may involve deleting a table.

375 """

376 # M2M fields are a special case

377 if field.many_to_many:

378 # For implicit M2M tables, delete the auto-created table

379 if field.remote_field.through._meta.auto_created:

380 self.delete_model(field.remote_field.through)

381 # For explicit "through" M2M fields, do nothing

382 elif (

383 self.connection.features.can_alter_table_drop_column

384 # Primary keys, unique fields, indexed fields, and foreign keys are

385 # not supported in ALTER TABLE DROP COLUMN.

386 and not field.primary_key

387 and not field.unique

388 and not field.db_index

389 and not (field.remote_field and field.db_constraint)

390 ):

391 super().remove_field(model, field)

392 # For everything else, remake.

393 else:

394 # It might not actually have a column behind it

395 if field.db_parameters(connection=self.connection)["type"] is None:

396 return

397 self._remake_table(model, delete_field=field)

398

399 def _alter_field(

400 self,

401 model,

402 old_field,

403 new_field,

404 old_type,

405 new_type,

406 old_db_params,

407 new_db_params,

408 strict=False,

409 ):

410 """Perform a "physical" (non-ManyToMany) field update."""

411 # Use "ALTER TABLE ... RENAME COLUMN" if only the column name

412 # changed and there aren't any constraints.

413 if (

414 self.connection.features.can_alter_table_rename_column

415 and old_field.column != new_field.column

416 and self.column_sql(model, old_field) == self.column_sql(model, new_field)

417 and not (

418 old_field.remote_field

419 and old_field.db_constraint

420 or new_field.remote_field

421 and new_field.db_constraint

422 )

423 ):

424 return self.execute(

425 self._rename_field_sql(

426 model._meta.db_table, old_field, new_field, new_type

427 )

428 )

429 # Alter by remaking table

430 self._remake_table(model, alter_fields=[(old_field, new_field)])

431 # Rebuild tables with FKs pointing to this field.

432 old_collation = old_db_params.get("collation")

433 new_collation = new_db_params.get("collation")

434 if new_field.unique and (

435 old_type != new_type or old_collation != new_collation

436 ):

437 related_models = set()

438 opts = new_field.model._meta

439 for remote_field in opts.related_objects:

440 # Ignore self-relationship since the table was already rebuilt.

441 if remote_field.related_model == model:

442 continue

443 if not remote_field.many_to_many:

444 if remote_field.field_name == new_field.name:

445 related_models.add(remote_field.related_model)

446 elif new_field.primary_key and remote_field.through._meta.auto_created:

447 related_models.add(remote_field.through)

448 if new_field.primary_key:

449 for many_to_many in opts.many_to_many:

450 # Ignore self-relationship since the table was already rebuilt.

451 if many_to_many.related_model == model:

452 continue

453 if many_to_many.remote_field.through._meta.auto_created:

454 related_models.add(many_to_many.remote_field.through)

455 for related_model in related_models:

456 self._remake_table(related_model)

457

458 def _alter_many_to_many(self, model, old_field, new_field, strict):

459 """Alter M2Ms to repoint their to= endpoints."""

460 if (

461 old_field.remote_field.through._meta.db_table

462 == new_field.remote_field.through._meta.db_table

463 ):

464 # The field name didn't change, but some options did, so we have to

465 # propagate this altering.

466 self._remake_table(

467 old_field.remote_field.through,

468 alter_fields=[

469 (

470 # The field that points to the target model is needed,

471 # so that table can be remade with the new m2m field -

472 # this is m2m_reverse_field_name().

473 old_field.remote_field.through._meta.get_field(

474 old_field.m2m_reverse_field_name()

475 ),

476 new_field.remote_field.through._meta.get_field(

477 new_field.m2m_reverse_field_name()

478 ),

479 ),

480 (

481 # The field that points to the model itself is needed,

482 # so that table can be remade with the new self field -

483 # this is m2m_field_name().

484 old_field.remote_field.through._meta.get_field(

485 old_field.m2m_field_name()

486 ),

487 new_field.remote_field.through._meta.get_field(

488 new_field.m2m_field_name()

489 ),

490 ),

491 ],

492 )

493 return

494

495 # Make a new through table

496 self.create_model(new_field.remote_field.through)

497 # Copy the data across

498 self.execute(

499 "INSERT INTO {} ({}) SELECT {} FROM {}".format(

500 self.quote_name(new_field.remote_field.through._meta.db_table),

501 ", ".join(

502 [

503 "id",

504 new_field.m2m_column_name(),

505 new_field.m2m_reverse_name(),

506 ]

507 ),

508 ", ".join(

509 [

510 "id",

511 old_field.m2m_column_name(),

512 old_field.m2m_reverse_name(),

513 ]

514 ),

515 self.quote_name(old_field.remote_field.through._meta.db_table),

516 )

517 )

518 # Delete the old through table

519 self.delete_model(old_field.remote_field.through)

520

521 def add_constraint(self, model, constraint):

522 if isinstance(constraint, UniqueConstraint) and (

523 constraint.condition

524 or constraint.contains_expressions

525 or constraint.include

526 or constraint.deferrable

527 ):

528 super().add_constraint(model, constraint)

529 else:

530 self._remake_table(model)

531

532 def remove_constraint(self, model, constraint):

533 if isinstance(constraint, UniqueConstraint) and (

534 constraint.condition

535 or constraint.contains_expressions

536 or constraint.include

537 or constraint.deferrable

538 ):

539 super().remove_constraint(model, constraint)

540 else:

541 self._remake_table(model)

542

543 def _collate_sql(self, collation):

544 return "COLLATE " + collation