Coverage for cc_modules/cc_exportrecipient.py: 58%

1"""

2 camcops_server/cc_modules/cc_exportrecipient.py

3

4===============================================================================

5

6 Copyright (C) 2012, University of Cambridge, Department of Psychiatry.

7 Created by Rudolf Cardinal (rnc1001@cam.ac.uk).

8

9 This file is part of CamCOPS.

10

11 CamCOPS is free software: you can redistribute it and/or modify

12 it under the terms of the GNU General Public License as published by

13 the Free Software Foundation, either version 3 of the License, or

14 (at your option) any later version.

15

16 CamCOPS is distributed in the hope that it will be useful,

17 but WITHOUT ANY WARRANTY; without even the implied warranty of

18 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

19 GNU General Public License for more details.

20

21 You should have received a copy of the GNU General Public License

22 along with CamCOPS. If not, see <https://www.gnu.org/licenses/>.

23

24===============================================================================

25

26**ExportRecipient class.**

27

28"""

29

30 import datetime

31 import logging

32 from typing import List, Optional, TYPE_CHECKING

33

34 from cardinal_pythonlib.logs import BraceStyleAdapter

35 from cardinal_pythonlib.reprfunc import simple_repr

36 from cardinal_pythonlib.sqlalchemy.list_types import (

37 IntListType,

38 StringListType,

39)

40 from cardinal_pythonlib.sqlalchemy.orm_inspect import gen_columns

41 from cardinal_pythonlib.sqlalchemy.session import get_safe_url_from_url

42 from sqlalchemy.event.api import listens_for

43 from sqlalchemy.orm import (

44 Mapped,

45 mapped_column,

46 reconstructor,

47 Session as SqlASession,

48)

49 from sqlalchemy.sql.sqltypes import (

50 BigInteger,

51 DateTime,

52 Text,

53)

54

55 from camcops_server.cc_modules.cc_exportrecipientinfo import (

56 ExportRecipientInfo,

57)

58 from camcops_server.cc_modules.cc_simpleobjects import TaskExportOptions

59 from camcops_server.cc_modules.cc_sqla_coltypes import (

60 EmailAddressColType,

61 ExportRecipientNameColType,

62 ExportTransmissionMethodColType,

63 FileSpecColType,

64 HostnameColType,

65 UrlColType,

66 UserNameExternalColType,

67)

68 from camcops_server.cc_modules.cc_sqlalchemy import Base

69

70 if TYPE_CHECKING:

71 from sqlalchemy.engine.base import Connection

72 from sqlalchemy.orm.mapper import Mapper

73 from camcops_server.cc_modules.cc_task import Task

74

75 log = BraceStyleAdapter(logging.getLogger(__name__))

76

77

78# =============================================================================

79# ExportRecipient class

80# =============================================================================

81

82

83 class ExportRecipient(ExportRecipientInfo, Base):

84 """

85 SQLAlchemy ORM class representing an export recipient.

86

87 This has a close relationship with (and inherits from)

88 :class:`camcops_server.cc_modules.cc_exportrecipientinfo.ExportRecipientInfo`

89 (q.v.).

90

91 Full details of parameters are in the docs for the config file.

92 """

93

94 __tablename__ = "_export_recipients"

95

96 IGNORE_FOR_EQ_ATTRNAMES = ExportRecipientInfo.IGNORE_FOR_EQ_ATTRNAMES + [

97 # Attribute names to ignore for equality comparison (is one recipient

98 # record functionally equal to another?).

99 "id",

100 "current",

101 "group_names", # Python only

102 ]

103 RECOPY_EACH_TIME_FROM_CONFIG_ATTRNAMES = [

104 # Fields representing sensitive information, not stored in the

105 # database. See also init_on_load() function.

106 "email_host_password",

107 "fhir_app_secret",

108 "fhir_launch_token",

109 "redcap_api_key",

110 ]

111

112 # -------------------------------------------------------------------------

113 # Identifying this object, and whether it's the "live" version

114 # -------------------------------------------------------------------------

115 id: Mapped[int] = mapped_column(

116 BigInteger,

117 primary_key=True,

118 autoincrement=True,

119 index=True,

120 comment="Export recipient ID (arbitrary primary key)",

121 )

122 recipient_name: Mapped[str] = mapped_column(

123 ExportRecipientNameColType,

124 comment="Name of export recipient",

125 )

126 current: Mapped[bool] = mapped_column(

127 default=False,

128 comment="Is this the current record for this recipient? (If not, it's "

129 "a historical record for audit purposes.)",

130 )

131

132 # -------------------------------------------------------------------------

133 # How to export

134 # -------------------------------------------------------------------------

135 transmission_method: Mapped[str] = mapped_column(

136 ExportTransmissionMethodColType,

137 comment="Export transmission method (e.g. hl7, file)",

138 )

139 push: Mapped[bool] = mapped_column(

140 default=False,

141 comment="Push (support auto-export on upload)?",

142 )

143 task_format: Mapped[Optional[str]] = mapped_column(

144 ExportTransmissionMethodColType,

145 comment="Format that task information should be sent in (e.g. PDF), "

146 "if not predetermined by the transmission method",

147 )

148 xml_field_comments: Mapped[bool] = mapped_column(

149 default=True,

150 comment="Whether to include field comments in XML output",

151 )

152

153 # -------------------------------------------------------------------------

154 # What to export

155 # -------------------------------------------------------------------------

156 all_groups: Mapped[bool] = mapped_column(

157 default=False,

158 comment="Export all groups? (If not, see group_ids.)",

159 )

160 group_ids: Mapped[Optional[list[int]]] = mapped_column(

161 IntListType,

162 comment="Integer IDs of CamCOPS group to export data from (as CSV)",

163 )

164 tasks: Mapped[Optional[list[str]]] = mapped_column(

165 StringListType,

166 comment="Base table names of CamCOPS tasks to export data from "

167 "(as CSV)",

168 )

169 start_datetime_utc: Mapped[Optional[datetime.datetime]] = mapped_column(

170 DateTime, comment="Start date/time for tasks (UTC)"

171 )

172 end_datetime_utc: Mapped[Optional[datetime.datetime]] = mapped_column(

173 DateTime, comment="End date/time for tasks (UTC)"

174 )

175 finalized_only: Mapped[bool] = mapped_column(

176 default=True,

177 comment="Send only finalized tasks",

178 )

179 include_anonymous: Mapped[bool] = mapped_column(

180 default=False,

181 comment="Include anonymous tasks? "

182 "Not applicable to some methods (e.g. HL7)",

183 )

184 primary_idnum: Mapped[int] = mapped_column(

185 comment="Which ID number is used as the primary ID?",

186 )

187 require_idnum_mandatory: Mapped[Optional[bool]] = mapped_column(

188 comment="Must the primary ID number be mandatory in the relevant "

189 "policy?",

190 )

191

192 # -------------------------------------------------------------------------

193 # Database

194 # -------------------------------------------------------------------------

195 db_url: Mapped[Optional[str]] = mapped_column(

196 UrlColType,

197 comment="(DATABASE) SQLAlchemy database URL for export",

198 )

199 db_echo: Mapped[bool] = mapped_column(

200 default=False,

201 comment="(DATABASE) Echo SQL applied to destination database?",

202 )

203 db_include_blobs: Mapped[bool] = mapped_column(

204 default=True,

205 comment="(DATABASE) Include BLOBs?",

206 )

207 db_add_summaries: Mapped[bool] = mapped_column(

208 default=True,

209 comment="(DATABASE) Add summary information?",

210 )

211 db_patient_id_per_row: Mapped[bool] = mapped_column(

212 default=True,

213 comment="(DATABASE) Add patient ID information per row?",

214 )

215

216 # -------------------------------------------------------------------------

217 # Email

218 # -------------------------------------------------------------------------

219 email_host: Mapped[Optional[str]] = mapped_column(

220 HostnameColType,

221 comment="(EMAIL) E-mail (SMTP) server host name/IP address",

222 )

223 email_port: Mapped[Optional[int]] = mapped_column(

224 "email_port",

225 comment="(EMAIL) E-mail (SMTP) server port number",

226 )

227 email_use_tls: Mapped[bool] = mapped_column(

228 default=True,

229 comment="(EMAIL) Use explicit TLS connection?",

230 )

231 email_host_username: Mapped[Optional[str]] = mapped_column(

232 UserNameExternalColType,

233 comment="(EMAIL) Username on e-mail server",

234 )

235 # email_host_password: not stored in database

236 email_from: Mapped[Optional[str]] = mapped_column(

237 EmailAddressColType,

238 comment='(EMAIL) "From:" address(es)',

239 )

240 email_sender: Mapped[Optional[str]] = mapped_column(

241 EmailAddressColType,

242 comment='(EMAIL) "Sender:" address(es)',

243 )

244 email_reply_to: Mapped[Optional[str]] = mapped_column(

245 EmailAddressColType,

246 comment='(EMAIL) "Reply-To:" address(es)',

247 )

248 email_to: Mapped[Optional[str]] = mapped_column(

249 Text,

250 comment='(EMAIL) "To:" recipient(s), as a CSV list',

251 )

252 email_cc: Mapped[Optional[str]] = mapped_column(

253 Text, comment='(EMAIL) "CC:" recipient(s), as a CSV list'

254 )

255 email_bcc: Mapped[Optional[str]] = mapped_column(

256 Text, comment='(EMAIL) "BCC:" recipient(s), as a CSV list'

257 )

258 email_patient_spec: Mapped[Optional[str]] = mapped_column(

259 "email_patient",

260 FileSpecColType,

261 comment="(EMAIL) Patient specification",

262 )

263 email_patient_spec_if_anonymous: Mapped[Optional[str]] = mapped_column(

264 FileSpecColType,

265 comment="(EMAIL) Patient specification for anonymous tasks",

266 )

267 email_subject: Mapped[Optional[str]] = mapped_column(

268 FileSpecColType,

269 comment="(EMAIL) Subject specification",

270 )

271 email_body_as_html: Mapped[bool] = mapped_column(

272 default=False,

273 comment="(EMAIL) Is the body HTML, rather than plain text?",

274 )

275 email_body: Mapped[Optional[str]] = mapped_column(

276 Text, comment="(EMAIL) Body contents"

277 )

278 email_keep_message: Mapped[bool] = mapped_column(

279 default=False,

280 comment="(EMAIL) Keep entire message?",

281 )

282

283 # -------------------------------------------------------------------------

284 # HL7

285 # -------------------------------------------------------------------------

286 hl7_host: Mapped[Optional[str]] = mapped_column(

287 HostnameColType,

288 comment="(HL7) Destination host name/IP address",

289 )

290 hl7_port: Mapped[Optional[int]] = mapped_column(

291 comment="(HL7) Destination port number"

292 )

293 hl7_ping_first: Mapped[bool] = mapped_column(

294 default=False,

295 comment="(HL7) Ping via TCP/IP before sending HL7 messages?",

296 )

297 hl7_network_timeout_ms: Mapped[Optional[int]] = mapped_column(

298 "hl7_network_timeout_ms",

299 comment="(HL7) Network timeout (ms).",

300 )

301 hl7_keep_message: Mapped[bool] = mapped_column(

302 default=False,

303 comment="(HL7) Keep copy of message in database? (May be large!)",

304 )

305 hl7_keep_reply: Mapped[bool] = mapped_column(

306 default=False,

307 comment="(HL7) Keep copy of server's reply in database?",

308 )

309 hl7_debug_divert_to_file: Mapped[bool] = mapped_column(

310 default=False,

311 comment="(HL7 debugging option) Divert messages to files?",

312 )

313 hl7_debug_treat_diverted_as_sent: Mapped[bool] = mapped_column(

314 default=False,

315 comment=(

316 "(HL7 debugging option) Treat messages diverted to file as sent"

317 ),

318 )

319

320 # -------------------------------------------------------------------------

321 # File

322 # -------------------------------------------------------------------------

323 file_patient_spec: Mapped[Optional[str]] = mapped_column(

324 FileSpecColType,

325 comment="(FILE) Patient part of filename specification",

326 )

327 file_patient_spec_if_anonymous: Mapped[Optional[str]] = mapped_column(

328 FileSpecColType,

329 comment=(

330 "(FILE) Patient part of filename specification for anonymous "

331 "tasks"

332 ),

333 )

334 file_filename_spec: Mapped[Optional[str]] = mapped_column(

335 FileSpecColType,

336 comment="(FILE) Filename specification",

337 )

338 file_make_directory: Mapped[bool] = mapped_column(

339 default=True,

340 comment=(

341 "(FILE) Make destination directory if it doesn't already exist"

342 ),

343 )

344 file_overwrite_files: Mapped[bool] = mapped_column(

345 default=False,

346 comment="(FILE) Overwrite existing files",

347 )

348 file_export_rio_metadata: Mapped[bool] = mapped_column(

349 default=False,

350 comment="(FILE) Export RiO metadata file along with main file?",

351 )

352 file_script_after_export: Mapped[Optional[str]] = mapped_column(

353 Text,

354 comment="(FILE) Command/script to run after file export",

355 )

356

357 # -------------------------------------------------------------------------

358 # File/RiO

359 # -------------------------------------------------------------------------

360 rio_idnum: Mapped[Optional[int]] = mapped_column(

361 "rio_idnum",

362 comment="(FILE / RiO) RiO metadata: which ID number is the RiO ID?",

363 )

364 rio_uploading_user: Mapped[Optional[str]] = mapped_column(

365 Text,

366 comment="(FILE / RiO) RiO metadata: name of automatic upload user",

367 )

368 rio_document_type: Mapped[Optional[str]] = mapped_column(

369 Text,

370 comment="(FILE / RiO) RiO metadata: document type for RiO",

371 )

372

373 # -------------------------------------------------------------------------

374 # REDCap export

375 # -------------------------------------------------------------------------

376 redcap_api_url: Mapped[Optional[str]] = mapped_column(

377 Text,

378 comment="(REDCap) REDCap API URL, pointing to the REDCap server",

379 )

380 redcap_fieldmap_filename: Mapped[Optional[str]] = mapped_column(

381 Text,

382 comment="(REDCap) File defining CamCOPS-to-REDCap field mapping",

383 )

384

385 # -------------------------------------------------------------------------

386 # FHIR export

387 # -------------------------------------------------------------------------

388 fhir_api_url: Mapped[Optional[str]] = mapped_column(

389 Text,

390 comment="(FHIR) FHIR API URL, pointing to the FHIR server",

391 )

392 fhir_app_id: Mapped[Optional[str]] = mapped_column(

393 Text,

394 comment="(FHIR) FHIR app ID, identifying CamCOPS as the data source",

395 )

396 fhir_concurrent: Mapped[Optional[bool]] = mapped_column(

397 default=False,

398 comment="(FHIR) Server supports concurrency (parallel processing)?",

399 )

400

401 def __hash__(self) -> int:

402 """

403 Used by the ``merge_db`` function, and specifically the old-to-new map

404 maintained by :func:`cardinal_pythonlib.sqlalchemy.merge_db.merge_db`.

405 """

406 return hash(f"{self.id}_{self.recipient_name}")

407

408 @reconstructor

409 def init_on_load(self) -> None:

410 """

411 Called when SQLAlchemy recreates an object; see

412 https://docs.sqlalchemy.org/en/latest/orm/constructors.html.

413

414 Sets Python-only attributes.

415

416 See also IGNORE_FOR_EQ_ATTRNAMES,

417 NEEDS_RECOPYING_EACH_TIME_FROM_CONFIG_ATTRNAMES.

418 """

419 self.group_names = [] # type: List[str]

420

421 # Within NEEDS_RECOPYING_EACH_TIME_FROM_CONFIG_ATTRNAMES:

422 self.email_host_password = ""

423 self.fhir_app_secret = ""

424 self.fhir_launch_token = None # type: Optional[str]

425 self.redcap_api_key = ""

426

427 def get_attrnames(self) -> List[str]:

428 """

429 Returns all relevant attribute names.

430 """

431 attrnames = set([attrname for attrname, _ in gen_columns(self)])

432 attrnames.update(

433 key for key in self.__dict__ if not key.startswith("_")

434 )

435 return sorted(attrnames)

436

437 def __repr__(self) -> str:

438 return simple_repr(self, self.get_attrnames())

439

440 def is_upload_suitable_for_push(

441 self, tablename: str, uploading_group_id: int

442 ) -> bool:

443 """

444 Might an upload potentially give tasks to be "pushed"?

445

446 Called by

447 :func:`camcops_server.cc_modules.cc_client_api_core.get_task_push_export_pks`.

448

449 Args:

450 tablename: table name being uploaded

451 uploading_group_id: group ID if the uploading user

452

453 Returns:

454 whether this upload should be considered further

455 """

456 if not self.push:

457 # Not a push export recipient

458 return False

459 if self.tasks and tablename not in self.tasks:

460 # Recipient is restricted to tasks that don't include the table

461 # being uploaded (or, the table is a subtable that we don't care

462 # about)

463 return False

464 if not self.all_groups:

465 # Recipient is restricted to specific groups

466 if uploading_group_id not in self.group_ids:

467 # Wrong group!

468 return False

469 return True

470

471 def is_task_suitable(self, task: "Task") -> bool:

472 """

473 Used as a double-check that a task remains suitable.

474

475 Args:

476 task: a :class:`camcops_server.cc_modules.cc_task.Task`

477

478 Returns:

479 bool: is the task suitable for this recipient?

480 """

481

482 def _warn(reason: str) -> None:

483 log.info(

484 "For recipient {}, task {!r} is unsuitable: {}",

485 self,

486 task,

487 reason,

488 )

489 # Not a warning, actually; it's normal to see these because it

490 # allows the client API to skip some checks for speed.

491

492 if self.tasks and task.tablename not in self.tasks:

493 _warn(f"Task type {task.tablename!r} not included")

494 return False

495

496 if not self.all_groups:

497 task_group_id = task.group_id

498 if task_group_id not in self.group_ids:

499 _warn(f"group_id {task_group_id} not permitted")

500 return False

501

502 if not self.include_anonymous and task.is_anonymous:

503 _warn("task is anonymous")

504 return False

505

506 if self.finalized_only and not task.is_preserved():

507 _warn("task not finalized")

508 return False

509

510 if self.start_datetime_utc or self.end_datetime_utc:

511 task_dt = task.get_creation_datetime_utc_tz_unaware()

512 if self.start_datetime_utc and task_dt < self.start_datetime_utc:

513 _warn("task created before recipient start_datetime_utc")

514 return False

515 if self.end_datetime_utc and task_dt >= self.end_datetime_utc:

516 _warn("task created at/after recipient end_datetime_utc")

517 return False

518

519 if not task.is_anonymous and self.primary_idnum is not None:

520 patient = task.patient

521 if not patient:

522 _warn("missing patient")

523 return False

524 if not patient.has_idnum_type(self.primary_idnum):

525 _warn(

526 f"task's patient is missing ID number type "

527 f"{self.primary_idnum}"

528 )

529 return False

530

531 return True

532

533 @classmethod

534 def get_existing_matching_recipient(

535 cls, dbsession: SqlASession, recipient: "ExportRecipient"

536 ) -> Optional["ExportRecipient"]:

537 """

538 Retrieves an active instance from the database that matches ``other``,

539 if there is one.

540

541 Args:

542 dbsession: a :class:`sqlalchemy.orm.session.Session`

543 recipient: an :class:`ExportRecipient`

544

545 Returns:

546 a database instance of :class:`ExportRecipient` that matches, or

547 ``None``.

548 """

549 # noinspection PyPep8

550 q = dbsession.query(cls).filter(

551 cls.recipient_name == recipient.recipient_name,

552 cls.current == True, # noqa: E712

553 )

554 results = q.all()

555 if len(results) > 1:

556 raise ValueError(

557 "Database has gone wrong: more than one active record for "

558 "{t}.{c} = {r}".format(

559 t=cls.__tablename__,

560 c=cls.recipient_name.name, # column name from Column

561 r=recipient.recipient_name,

562 )

563 )

564 if results:

565 r = results[0]

566 if recipient == r:

567 return r

568 return None

569

570 @property

571 def db_url_obscuring_password(self) -> Optional[str]:

572 """

573 Returns the database URL (if present), but with its password obscured.

574 """

575 if not self.db_url:

576 return self.db_url

577 return get_safe_url_from_url(self.db_url)

578

579 def get_task_export_options(self) -> TaskExportOptions:

580 return TaskExportOptions(

581 xml_include_comments=self.xml_field_comments,

582 xml_with_header_comments=self.xml_field_comments,

583 )

584

585

586# noinspection PyUnusedLocal

587@listens_for(ExportRecipient, "after_insert")

588@listens_for(ExportRecipient, "after_update")

589 def _check_current(

590 mapper: "Mapper", connection: "Connection", target: ExportRecipient

591) -> None:

592 """

593 Ensures that only one :class:`ExportRecipient` is marked as ``current``

594 per ``recipient_name``.

595

596 As per

597 https://stackoverflow.com/questions/6269469/mark-a-single-row-in-a-table-in-sqlalchemy.

598 """

599 if target.current:

600 # noinspection PyUnresolvedReferences

601 connection.execute(

602 ExportRecipient.__table__.update() # type: ignore[attr-defined]

603 .values(current=False)

604 .where(ExportRecipient.recipient_name == target.recipient_name)

605 .where(ExportRecipient.id != target.id)

606 )