import re
import unittest
from ir_datasets.datasets.cord19 import Cord19Doc, Cord19FullTextDoc, Cord19FullTextSection
from ir_datasets.formats import TrecQrel, TrecQuery
from .base import DatasetIntegrationTest


class TestCord19(DatasetIntegrationTest):
    def test_cord19_docs(self):
        self._test_docs('cord19', count=192509, items={
            0: Cord19Doc('ug7v899j', 'Clinical features of culture-proven Mycoplasma pneumoniae infections at King Abdulaziz University Hospital, Jeddah, Saudi Arabia', '10.1186/1471-2334-1-6', '2001-07-04', re.compile('^OBJECTIVE: This retrospective chart review describes the epidemiology and clinical features of 40 pa.{1647}preschool children and that the mortality rate of pneumonia in patients with comorbidities was high\\.$', flags=48)),
            9: Cord19Doc('jg13scgo', 'Technical Description of RODS: A Real-time Public Health Surveillance System', '10.1197/jamia.m1345', '2003-09-01', re.compile('^This report describes the design and implementation of the Real\\-time Outbreak and Disease Surveillan.{1077} be a resource for implementing, evaluating, and applying new methods of public health surveillance\\.$', flags=48)),
            192508: Cord19Doc('pnl9th2c', 'Vascular Life during the COVID-19 Pandemic Reminds Us to Prepare for the Unexpected', '10.1016/j.ejvs.2020.04.040', '2020-05-12', ''),
        })
        self._test_docs('cord19/fulltext', count=192509, items={
            0: Cord19FullTextDoc('ug7v899j', 'Clinical features of culture-proven Mycoplasma pneumoniae infections at King Abdulaziz University Hospital, Jeddah, Saudi Arabia', '10.1186/1471-2334-1-6', '2001-07-04', re.compile('^OBJECTIVE: This retrospective chart review describes the epidemiology and clinical features of 40 pa.{1647}preschool children and that the mortality rate of pneumonia in patients with comorbidities was high\\.$', flags=48), (Cord19FullTextSection(title='Introduction', text='Mycoplasma pneumoniae is a common cause of upper and lower respiratory tract infections. It remains one of the most frequent causes of atypical pneumonia particularly among young adults.[1,2,3,4,5] Although it is highly transmissible, most infections caused by this organism are relatively minor and include pharyngitis, tracheobronchitis, bronchiolitis, and croup with one fifth of infections being asymptomatic.[6,7] Only 3 -10% of infected subjects develop symptoms consistent with bronchopneumonia and mortality from infection is rare.[6,7] The organism is fastidious and difficult to grow on cultures. Therefore, diagnosis of infections caused by this organism is usually confirmed with serological tests or polymerase chain reaction-gene amplification techniques. At King Abdulaziz University Hospital (KAUH), Jeddah, Saudi Arabia, the facility to perform Mycoplasma culture has been available since January 1997. As published information concerning M. pneumoniae infections in Saudi Arabia is scarce,[8,9,10] we wished to study the epidemiology and clinical features of culture-proven infections caused by this organism at this hospital.'), Cord19FullTextSection(title='Institution and patient population ::: Methods', text='KAUH is a tertiary care teaching hospital with a bed capacity of 265 beds and annual admissions of 18000 to 19000 patients. Patients with M. pneumoniae positive cultures from respiratory specimens were identified over a 24-months" period from January, 1997 through December, 1998 for this review.'), Cord19FullTextSection(title='Data collection ::: Methods', text="During the study period, respiratory specimens (sputum, nasopharyngeal aspiration, endotracheal secretion, and bronchoalveolar lavage) for M. pneumoniae culture were obtained from patients with upper or lower respiratory tract infections seen as inpatients or in the outpatient or emergency departments. Respiratory specimens were aslo Gram-stained and cultured for bacteria and viruses. M. pneumoniae serological tests for IgG or IgM were not available at KAUH during the study period. All positive culture results were obtained from the Microbiology laboratory records. Charts of patients were reviewed with standardized data collection. Information collected included patients' demographics, comorbidities, clinical manifestations, complications, and outcome."), Cord19FullTextSection(title='Microbiological methods ::: Methods', text='M. pneumoniae was cultured using the classic M. pneumoniae agar medium (M.P. agar) and the Pneumofast tray (Pneumofast®, International Microbio, Signes, France). Specimens were processed according to the instructions of the manufacturer. The M.P. agars and Pneumofast trays were incubated anaerobically at 37°C and inspected daily for 4 weeks. The organism was identified based on typical colonial morphology (granular colonies, rarely fried-egg-like, 10-150 ∝ in diameter) on the M.P. agar medium and the change in the Pneumofast broth color from red to orange then to yellow (glucose fermentation) in the absence of turbidity of the broth. Antibiotic sensitivity profile on the Pneumofast tray was also used for identification according to the instructions of the manufacturer. Bacterial and viral cultures were performed using standard methods.'), Cord19FullTextSection(title='Definitions ::: Methods', text='M. pneumoniae isolates were considered community-acquired if they were recovered from unhospitalized patients or within 72 hours of admission to the hospital, and nosocomial if they were recovered beyond that period.'), Cord19FullTextSection(title='Definitions ::: Methods', text='Pneumonia was diagnosed based on clinical symptoms and signs, along with radiographic evidence of pneumonia when possible. Severe pneumonia was defined as pneumonia associated with tachycardia (>140 /minute), tachypnoea (>30/minute), hypotension (Systolic blood pressure <90 mmHg), hypoxemia (arterial oxygen partial pressure <8 kPa or oxygen saturation <90%), and/or more than 2 areas of consolidation.'), Cord19FullTextSection(title='Definitions ::: Methods', text='Outcome of patients with M. pneumoniae infection was classified into 4 categories; uneventful recovery, recovery following complications, death due to M. pneumoniae infection, or death unrelated to M. pneumoniae infection.'), Cord19FullTextSection(title='Data Analysis', text="The Statistical Package for Social Sciences (SPSS) program was used for data analysis. Comparison of categorical data was by Chi-square statistic or Fisher's exact test for small expected values."), Cord19FullTextSection(title='Results', text="A total of 40 respiratory specimens from 40 patients were positive for M. pneumoniae over the 24-months study period. The demographic and epidemiological characteristics of the patients are summarized in Table 1. Of all isolates, 37 (92.5%) were community-acquired and 3 (7.5%) were nosocomial. Thirty-three (82.5%) patients required admission to the hospital and the remaining 7 (17.5%) were treated as outpatients. Twenty-four isolates (60%) were associated with pneumonia, 14 (35%) with upper respiratory tract infections, and 2 (5%) with bronchiolitis. Of the 24 cases of pneumonia, 21 were confirmed radiologically and the remaining 3 were diagnosed clinically. The two cases of bronchiolitis occurred in 2 children, one and three years old. Thirty-one patients (77.5%) had comorbidities. Eleven patients (27.5%) had cardiopulmonary comorbidities (asthma, 8, lung fibrosis, 1, congestive heart failure, 1, congenial heart disease, 1), 9 patients (22.5%) were immunocompromised (malignancy, 7, steroid therapy, 3, Human immunodeficiency virus infection, 1), and 11 patients (27.5%) had other comorbidities (premature newborns, 2, and one each of myelodysplastic syndrome, myeloproliferative disorder, sickle cell anemia, Evan's syndrome, Down syndrome, sarcoidosis, demyelinating disease, cerebral palsy, and spinal muscle atrophy). Organisms concomitantly isolated with M. pneumoniae from the respiratory tract included herpes simplex virus type 1 (2 occasions), adenovirus (2 occasions), cytomegalo virus (1 occasion), respiratory syncytial virus (1 occasion), and bacterial isolates (2 occasions: Acinetobacter species, 1, and Enter obacter cloacae, 1)."), Cord19FullTextSection(title='Results', text='Clinical manifestations associated with M. pneumoniae infections are summarized in Table 2. Pneumonia was more common than upper respiratory tract infections (57.5 % versus 27.5%, respectively). Immunocompromised patients were more likely to present with pneumonia as opposed to upper respiratory tract infection or bronchiolitis than non-immunocompromised patients (8/9 versus 16/31, P = 0.05). Similarly, there was a tendency for patients 60 years of age or older to present with pneumonia more frequently than those below 60 (4/4 versus 20/36, P = 0.1). Of the 24 patients with clinically or radiologically confirmed pneumonia, 19 (79.2%) had crepitations and only 6 (25%) had bronchial breath sounds on physical examination. Of the 16 patients in whom wheezes were detected, 9 (56.3%) were not known to have asthma or other obstructive airway disease.'), Cord19FullTextSection(title='Results', text='Complications and outcome of patients are shown in Table 3. Of the 24 patients with pneumonia, 21 (87.5%) were admitted to the hospital, and 20 (83.3%) had comorbidities. All patients with upper respiratory tract infections (11 patients) or bronchiolitis (2 patients) had uneventful recovery. Of the 24 patients with pneumonia, 14 (58.3%) had uneventful recovery, 4 (16.7%) recovered following some complications (acute respiratory distress syndrome, 2, respiratory failure, 1, septic shock, 1), 3 (12.5%) died because of M pneumoniae infection, and 3 (12.5%) died due to underlying comorbidities. The 3 patients who died of M pneumoniae pneumonia had other comorbidities; one had congestive heart failure, the second had congenital heart disease, and the third was a 3-months old infant born prematurely at 32 weeks of gestation who previously had 3 episodes of pneumonia due to other pathogens.'), Cord19FullTextSection(title='Discussion', text='Mycoplasma pneumoniae is one of the most common causes of atypical pneumonia accounting for 5-23% of community-acquired pneumonia,[1,2,3,4,5] In a study of 511 children with acute respiratory tract infection in Riyadh, Saudi Arabia, Mycoplasma pneumoniae was found to be the second most common causative agent after Respiratory syncytial virus (RSV) accounting for 9% of all cases,[8] In a study of 112 adult patients with community acquired pneumonia admitted to a military hospital in Riyadh, Saudi Arabia, this organism accounted for 6% of all cases,[9] In another retrospective study of 567 pneumonic episodes in adult patients from Al-Qassim area, the organism accounted for 23% of all episodes,[10] The organism also causes other relatively minor infections such as pharyngitis, tracheobronchitis, bronchiolitis, and croup. It is transmitted from person-to-person by infected respiratory droplets during close contact. It is most common in school-aged children, military recruits, and college students.[11] Most cases occur singly or as family outbreaks. Larger outbreaks can also occur in closed populations such as military recruit camps or boarding schools,[12] Infection occurs most frequently during the fall and winter in temperate climates but may develop year-round,[13] The average incubation period is 3 weeks following exposure,[6] Although rare, complications are protean and may involve virtually any organ system such as the respiratory system (e.g.: pleurisy, pneumothorax, acute respiratory distress syndrome, lung abscess), the hematologic system (e.g.: hemolytic anemia, intravascular coagulation, thrombocytopenia), the dermatologic system (e.g.: maculopapular or urticarial rashes, erythema multiforme, erythema nodosum), the musculoskeletal system (e.g.: myalgias, arthralgias, arthritis), the cardiovascular system (e.g.: pericarditis, myocarditis), the nervous system (e.g.: meningoencephalitis, Guillain-Barre syndrome, neuropathies, acute psychosis), or the eye (optic disc edema, optic nerve atrophy, retinal exudation and hemorrhages).[6,7,14,15,16,17,18] Immunity following infection is not long lasting.[11]'), Cord19FullTextSection(title='Discussion', text='In our study, the infection affected all age groups but was most common in infants (32.5%) and preschool children (22.5%), and least common in adults aged 15 to 30 years (2.5%) and elderly above 70 years of age (5%). This contrasts with data from temperate countries where the infection is most common in school-aged children, and young adults.[11] One possible explanation for this difference is that infants and preschool children perhaps had more severe infections than did school-aged children, and young adults which prompted presentation of the former group to the hospital. The infection occurred year-round but was most common in the fall (35%), and spring (30%), and least common in the summer (10%). Most infections were community-acquired (92.5%).'), Cord19FullTextSection(title='Discussion', text='More than one half of patients (57.5%) presented with pneumonia, and about a third (27.5%) presented with upper respiratory tract infection, Immunocompromised patients and patients 60 years of age or older were more likely to present with pneumonia as opposed to upper respiratory tract infection than non-immunocompromised patients or those below 60 years of age. Cough (82.5%), fever (75%), and malaise (58.8%) were the most common presenting symptoms. Cough was usually dry or slightly productive of white sputum and mild to moderate in severity. Most febrile patients had mild to moderate fever of 39°C or less; high-grade fever of more than 39°C was rare. Crepitations (60%), and wheezes (40%) were the most common signs. Wheezes were as common in patients with no history of obstructive airway disease (9 patients) as it was in those with such a history (7 patients). Bronchial breathing as a sign of consolidation was detected in only one fourth of patients with pneumonia, which is consistent with the known disparity between clinical and radiological signs of M pneumoniae pneumonia. Crepitations, however, were detected in the majority (79.2%) of patients. Pleuritic chest pain and pleural effusion were rare.'), Cord19FullTextSection(title='Discussion', text='More than half (56.5%) of the patients with pneumonia had uneventful recovery. Mortality from M. pneumoniae pneumonia was high (12.5%) and occurred only in patients with underlying comorbidities. None of the 9 patients with no underlying comorbidities died of M pneumoniae pneumonia. The relatively high complications rate (16.7%) and mortality (12.5%) related to M. pneumoniae pneumonia are likely due to selection bias as most patients with pneumonia were sick enough to require admission to the hospital (21/24 or 87.5%) and most of them had comorbidities (20/24 or 83.3%).'), Cord19FullTextSection(title='Discussion', text='In conclusion, our data shed some light on the epidemiology and clinical features of M pneumoniae infections in one of the Saudi tertiary care centers. The data are comparable to those of other countries except for the finding that infections were more common in infants and preschool children than in school children and young adults. Additionally, mortality attributable to M. pneumoniae pneumonia was relatively high in patients with comorbidities. It is hoped this information will assist clinicians in their approach and management of respiratory tract infections.'), Cord19FullTextSection(title='Pre-publication history', text='The pre-publication history for this paper can be accessed here:'), Cord19FullTextSection(title='Pre-publication history', text=''))),
            9: Cord19FullTextDoc('jg13scgo', 'Technical Description of RODS: A Real-time Public Health Surveillance System', '10.1197/jamia.m1345', '2003-09-01', re.compile('^This report describes the design and implementation of the Real\\-time Outbreak and Disease Surveillan.{1077} be a resource for implementing, evaluating, and applying new methods of public health surveillance\\.$', flags=48), (Cord19FullTextSection(title='Public Health Surveillance ::: Background', text='The role of public health surveillance is to collect, analyze, and interpret data about biological agents, diseases, risk factors, and other health events and to provide timely dissemination of collected information to decision makers.17 Conventionally, public health surveillance relies on manual operations and off-line analysis.'), Cord19FullTextSection(title='Syndromic Surveillance ::: Background', text="Existing syndromic surveillance systems include the CDC's drop-in surveillance systems,8 Early Notification of Community-based Epidemics (ESSENCE),10,18 the Lightweight Epidemiology Advanced Detection and Emergency Response System (LEADERS),19 the Rapid Syndrome Validation Project (RSVP),20 and the eight systems discussed by Lober et al.11"), Cord19FullTextSection(title='Syndromic Surveillance ::: Background', text="Lober et al. summarized desirable characteristics of syndromic surveillance systems and analyzed the extent to which systems that were in existence in 2001 had those characteristics.11 A limitation of most systems (e.g., ESSENCE,10 Children's Hospital in Boston,11 University of Washington11) was batch transfer of data, which may delay detection by as long as the time interval (periodicity) between batch transfers. For example, a surveillance system with daily batch transfer may delay by one day the detection of an outbreak."), Cord19FullTextSection(title='Syndromic Surveillance ::: Background', text="Some systems required manual data input (e.g., CDC's drop-in surveillance systems, RSVP,20 and LEADERS19), which is labor-intensive and, in the worst case, requires round-the-clock staffing. Manual data input is not a feasible mid- or long-term solution even if the approach is to add items to existing encounter forms (where the items still may be ignored by busy clinicians)."), Cord19FullTextSection(title='Syndromic Surveillance ::: Background', text='A third limitation for existing surveillance systems is that the systems may not exploit existing standards or communication protocols like Heath Level 7 (HL7) even when they are available.'), Cord19FullTextSection(title='Syndromic Surveillance ::: Background', text='The data type most commonly used among surveillance systems is symptoms or diagnoses of patients from ED and/or physician office visits. Other types of data identified in that study include emergency call center and nurse advice lines. Other types of data being used include sales of over-the-counter health care products, prescriptions, telephone call volumes to health care providers and drug stores, and absenteeism. We have conducted studies demonstrating that the free-text chief complaint data that we use correlate with outbreaks.21,22'), Cord19FullTextSection(title='Design Objectives', text="The overall design objective for RODS is similar to that of an early warning system for missile defense; namely, to collect whatever data are required to achieve early detection from as wide an area as necessary and to analyze the data in a way that they can be used effectively by decision makers. It is required that this analysis be done in close to real time. This design objective is complex and difficult to operationalize because of the large number of organisms and the even larger number of possible routes of dissemination all requiring potentially different types of data for their detection, different algorithms, and different time urgencies. For this reason, our focus since beginning the project in 1999 has been on the specific problem of detecting a large-scale outbreak due to an outdoor (outside buildings) aerosol release of anthrax. Additional design objectives were adherence to NEDSS standards to ensure future interoperability with other types of public health surveillance systems, scalability, and that the system could not rely on manual data entry, except when it was done in a focused way in response to the system's own analysis of passively collected data."), Cord19FullTextSection(title='Overview ::: Technical Description', text="RODS uses clinical data that are already being collected by health care providers and systems during the registration process. When a patient arrives at an ED (or an InstaCare in Utah), the registration clerk or triage nurse elicits the patient's reason for visit (i.e., the chief complaint), age, gender, home zip code, and other data and enter the data in a registration computer. The registration computer then generates an HL7 ADT (admission, discharge, and transfer) message and transmits it to the health system's HL7 message router (also called an integration engine). There usually is only one message router per health system even if there are many hospitals and facilities. These processes are all routine existing business activities and do not need to be created de novo for public health surveillance."), Cord19FullTextSection(title='Overview ::: Technical Description', text="▶ shows the flow of clinical data to and within RODS. The hospital's HL7 message router, upon receipt of an HL7 message from a registration computer, deletes identifiable information from the message and then transmits it to RODS over a secure virtual private network (VPN), or a leased line, or both (during the 2002 Winter Olympics we utilized both types of connections to each facility for fault tolerance). The RODS HL7 listener maintains the connection with the health system's message router and parses the HL7 message as described in more detail below. It then passes the chief complaint portion of the message to a Bayesian text classifier that assigns each free-text chief complaint to one of seven syndromic categories (or to an eighth category, other). The database stores the category data, which then are used by applications such as detection algorithms and user interfaces.  "), Cord19FullTextSection(title='Overview ::: Technical Description', text='Data about sales of OTC health care products are processed separately by the National Retail Data Monitor, which is discussed in detail in another article in this issue of JAMIA.23 The processing was kept separate intentionally because, in the future, the servers for the National Retail Data Monitor may operate in different physical locations than RODS. The RODS user interfaces can and do display sales of OTC health care products as will be discussed, but other user interfaces can be connected to the National Retail Data Monitor as well.'), Cord19FullTextSection(title='Data Sharing Agreements ::: Data Level ::: Technical Description', text='Prior to September 2001, RODS received data only from hospitals associated with the UPMC Health System, and efforts to recruit other hospitals met with resistance. After the terrorist attacks (including anthrax) in the Fall of 2001, other hospitals agreed to participate. Although data in this project are de-identified, certain information such as the number of ED visits by zip code were considered proprietary information by some health systems. Health Insurance Portability and Accountability Act (HIPAA) concerns also were very prominent in the discussions. Data-sharing agreements were executed with every participating health system that addressed these concerns. As an additional precaution, all RODS project members meet annually with University of Pittsburgh council to review obligations and are required to sign an agreement every year stating that they understand the terms of the data-sharing agreements and agree to abide by the terms. RODS began as a research project at the University of Pittsburgh in 1999 and has functioned with IRB approvals since that time.'), Cord19FullTextSection(title='Data Types ::: Data Level ::: Technical Description', text='Health care facilities send admission, discharge, and transfer (ADT) HL7 messages to RODS for patient visits in EDs and walk-in clinics. A minimal data set is sent, as shown in ▶, which qualifies as a HIPAA Limited Data Set.24 Currently the data elements are age (without date of birth), gender, home zip code, and free-text chief complaint.  '), Cord19FullTextSection(title='Data Transmission ::: Data Level ::: Technical Description', text='The HL7 listener receives HL7 messages from the message routers located in each health system. The HL7 listener then passes the received HL7 message to the HL7 parser bean, an Enterprise JavaBean (EJB) in the RODS business logic tier. The HL7 parser bean uses regular expressions to parse the fields in an HL7 message. The HL7 parser bean then stores the parsed elements into a database through a managed database connection pool.'), Cord19FullTextSection(title='Data Transmission ::: Data Level ::: Technical Description', text="Although nearly all health systems utilize the HL7 messaging standard, the location of individual data elements in an HL7 message may differ from health system to health system. For example, some care providers' systems record free-text chief complaint in the DG1 segment instead of the PV2 segment of an HL7 message. To resolve this mapping problem, a configuration file written in eXtensible Markup Language (XML), a standard protocol often used to define hierarchical data elements, defines where each of the data elements can be found in the HL7 message. When an HL7 listener starts up, it reads the hospital-dependent configuration file and passes the configuration information to the parser bean."), Cord19FullTextSection(title='Data Transmission ::: Data Level ::: Technical Description', text='We also use this configuration file to define the database table and field in which the HL7 parser bean should store each data element. This approach is useful because it allows the HL7 data to be stored to an external database. We anticipate that health departments with existing NEDSS or other public health surveillance databases may wish to use just this component of RODS for real-time collection of clinical data.'), Cord19FullTextSection(title='Data Transmission ::: Data Level ::: Technical Description', text='For hospitals that do not have HL7 message routers (two of approximately 60 in our experience to date), RODS accepts ED registration data files through either a secure Web-based data upload interface or a secure file transfer protocol. In general, these types of data transfers are technically trivial and for that reason are used by many groups but do not have the reliability of a HL7 connection (and have very undesirable time latencies).'), Cord19FullTextSection(title='Data Integrity ::: Data Level ::: Technical Description', text='RODS checks the integrity of the data in the HL7 messages that it receives. This processing is necessary because hospital data flows may have undesirable characteristics such as duplicates. RODS identifies and deletes duplicates by using a database trigger that creates a composite primary key before inserting the data. RODS also filters out scheduling messages, which are identified by the fact that they have future admitted date and time.'), Cord19FullTextSection(title='Data Integrity ::: Data Level ::: Technical Description', text="RODS monitors all data feeds to ensure continuous connections with health systems. If RODS does not receive data for six hours, it sends an alert to the RODS administrator and the sending health system's administrator. Because the commercial message routers that hospitals use queue up HL7 messages when encountering networking or system problems, data integrity is preserved."), Cord19FullTextSection(title='Database ::: Data Level ::: Technical Description', text='RODS uses an Oracle8i database to store ED registration data. (Oracle, Redwood Shores, CA). To ensure fast response for an online query (e.g., the daily counts of respiratory syndrome in a county for the past six months), we developed a cache table scheme that pre-aggregates counts and refreshes them every 30 minutes.25'), Cord19FullTextSection(title='Network Level ::: Technical Description', text='The communications network between RODS and health care systems consists of virtual private networks (VPN) and leased lines. RODS uses multivendor site-to-site Internet Protocol Security (IPSEC) VPNs to receive HL7 messages. During the Winter Olympics, we exclusively used leased lines for the primary connection because of concerns about possible communications interruptions due to Internet traffic related to the games. The leased lines consisted of a redundant pair of 128k fractional T1 lines. After the Olympics, we returned to use of VPNs, and RODS has operated reliably using VPNs in both Utah and Pennsylvania. The leased-line modality is used only to connect the Siemens Medical Systems Data Center with RODS for the transmission of data from nine health systems that are hosted by Siemens.'), Cord19FullTextSection(title='System Hardware ::: Technical Description', text='For connectivity with the HL7 message routers, we utilize hardware-based routers. The VPN router is a Cisco PIX 501 and the leased-line routers are a pair of Cisco 2600s (Cisco Systems, Inc., San Jose, CA).'), Cord19FullTextSection(title='System Hardware ::: Technical Description', text='All of the RODS processes can be run on a single computer, but in our current implementation—serving Pennsylvania and Utah as an application service provider—we use five dedicated servers: firewall, database, Web server, a geographic information system (GIS) server, and computation. The processes are written in Java code and can run on most platforms, but here we describe the specific platforms we use to indicate approximate sizing and processing requirements.'), Cord19FullTextSection(title='System Hardware ::: Technical Description', text='The database server is a Sun Microsystems Enterprise 250 configured with two Ultrasparc II 400Mhz processors, 2 gigabytes of RAM, and 36 gigabytes of mirrored hard drive space running an Oracle 8.1.7 (database) on Solaris 8 (Sun Microsystems, Inc., Santa Clara, CA).'), Cord19FullTextSection(title='System Hardware ::: Technical Description', text='The Web server is a Dell Poweredge 1550 configured with two 1Ghz Pentium III processors, 1 gigabyte of RAM, and 36 gigabytes of Redundant Arrays of Inexpensive Disk 5 (RAID-5) storage running Apache 1.3.24 (Web server), and Jboss 3.0 (described below in Fault Tolerance) on Redhat Linux 7.1 (Dell Computer Corporation, Round Rock, TX; Jboss Group, Atlanta, GA; Red Hat, Raleigh, NC).'), Cord19FullTextSection(title='System Hardware ::: Technical Description', text='The GIS server is a Dell Poweredge 350 configured with one 1Ghz Pentium III processor, 512 megabytes of RAM, and 18 gigabytes of storage running ArcIMS 4.0 (ESRI, Inc., Redlands, CA), an Internet-enabled geographic information system on Redhat Linux 7.3.'), Cord19FullTextSection(title='System Hardware ::: Technical Description', text='The computation server is a Penguin Computing server configured with dual Athlon MP 2400s, 1 gigabyte of RAM, and 750 gigabytes of RAID-5 storage running Oracle 9i on Redhat Linux 7.3.'), Cord19FullTextSection(title='System Hardware ::: Technical Description', text='Backup is performed nightly on all machines using a Sun StoreEdge L9 Tape Autoloader attached to the database server and Veritas Netbackup software (Veritas, Mountain View, CA).'), Cord19FullTextSection(title='Natural Language Processing ::: Application Level ::: Technical Description', text='RODS uses a naive Bayesian classifier called Complaint Coder (CoCo) to classify free-text chief complaints into one of the following syndromic categories: constitutional, respiratory, gastrointestinal, neurological, botulinic, rash, hemorrhagic, and other. CoCo computes the probability of each category, conditioned on each word in a free-text chief complaint and assigns a patient to the category with the highest probability.27 The probability distributions used by CoCo are learned from a manually created training set. CoCo can be retrained with local data, and it can be trained to detect a different set of syndromes than we currently use. CoCo runs as a local process on the RODS database server. CoCo was developed at the University of Pittsburgh and is available for free download at <http://health.pitt.edu/rods/sw>.'), Cord19FullTextSection(title='Detection Algorithms ::: Application Level ::: Technical Description', text='Over the course of the project, RODS has used two detection algorithms. These algorithms have not been formally field tested because the emphasis of the project to date has been on developing the data collection infrastructure more than field testing of algorithms.'), Cord19FullTextSection(title='Detection Algorithms ::: Application Level ::: Technical Description', text='The Recursive-Least-Square (RLS) adaptive filter28 currently runs every four hours, and alerts are sent to public health officials in Utah and Pennsylvania. RLS, a dynamic autoregressive linear model, computes an expected count for each syndrome category for seven counties in Utah and 16 counties in Pennsylvania as well as for the combined counts for each state. We use RLS because it has a minimal reliance on historical data for setting model parameters and a high sensitivity to rapid increases in a time series e.g., a sudden increase in daily counts. RLS triggers an alert when the current actual count exceeds the 95% confidence interval for the predicted count.'), Cord19FullTextSection(title='Detection Algorithms ::: Application Level ::: Technical Description', text="During the 2002 Olympics we also used the What's Strange About Recent Events (WSARE 1.0) algorithm.29 WSARE performs a heuristic search over combinations of temporal and spatial features to detect anomalous densities of cases in space and time. Such features include all aspects of recent patient records, including syndromal categories, age, gender, and geographical information about patients. The criteria used in the past for sending a WSARE 1.0 alert was that there has been an increase in the number of patients with specific characteristics relative to the counts on the same day of the week during recent weeks and the p-value after careful adjustment for multiple testing for the increase was ≤0.05. Version 3.0 of WSARE, which will incorporate a Bayesian model for computing expected counts rather than using unadjusted historical counts currently, is under development."), Cord19FullTextSection(title='Alert Notification ::: Application Level ::: Technical Description', text="When an algorithm triggers an alert based on the above criteria, RODS sends e-mail and/or page alerts to its users. RODS uses an XML-based configuration file to define users' e-mail and pager addresses. The e-mail version of the alert includes a URL link to a graph of the time series that triggered the alarm with two comparison time series: total visits for the same time period and normalized counts."), Cord19FullTextSection(title='User Interface ::: Application Level ::: Technical Description', text="RODS has a password-protected, encrypted Web site at which users can review health care registration and sales of OTC health care products on epidemic plots and maps. When a user logs in, RODS will check the user's profile and will display data only for his or her health department's jurisdiction. The interface comprises three screens—Main, Epiplot, and Mapplot."), Cord19FullTextSection(title='User Interface ::: Application Level ::: Technical Description', text='The main screen alternates views automatically among each of the available data sources (currently health care registrations and OTC products in Pennsylvania and Utah and OTC sales only for other states). The view alternates every two minutes as shown in ▶. The clinic visits view shows daily total visits and seven daily syndromes for the past week. The OTC data view shows daily sales for five product categories and the total, also for the past week. Users also can set the view to a specific county in a state. If the normalize control box is checked, the counts in the time series being displayed will be divided by (normalized by) the total daily sales of OTC health care products or ED visits for the region.  '), Cord19FullTextSection(title='User Interface ::: Application Level ::: Technical Description', text='The Epiplot screen provides a general epidemic plotting capability. The user can simultaneously view a mixture of different syndromes and OTC product categories for any geographic region (state, county, or zip code), and for any time interval. The user also can retrieve case details as shown in ▶. The Get Cases button queries the database for the admission date, age, zip code, and chief complaint (verbatim, not classified into syndrome category) of all patients in the time interval and typically is used to examine an anomalous density (spike) of cases. The Download Data button will download data as a compressed comma separated file for further analyses.  '), Cord19FullTextSection(title='User Interface ::: Application Level ::: Technical Description', text='The Mapplot screen is an interface to ArcIMS, an Internet-enabled GIS product developed by Environmental Systems Research Institute, Inc. Mapplot colors zip code regions to indicate the proportion of patients presenting with a particular syndrome. The GIS server also can overlay state boundaries, county boundaries, water bodies, hospital locations, landmarks, streets, and highways on the public health data as shown in ▶. Similar to Epiplot, Mapplot also can display case details for a user-selected zip code.  '), Cord19FullTextSection(title='Fault Tolerance ::: Technical Description', text='RODS has been in operation for four years and, like most production systems, has acquired many fault-tolerant features. For example, at the software level, HL7 listeners continue to receive messages and temporarily store the messages when the database is off-line. A data manager program runs every ten minutes and, on finding such a cache, it loads the unstored messages to the database when the database is back on-line. In addition, the data manager program monitors and restarts HL7 listeners as necessary. The database uses “archive log” mode to log every transaction to ensure that the database can recover from a system failure.'), Cord19FullTextSection(title='Fault Tolerance ::: Technical Description', text='The hardware architecture also is fault tolerant. All servers have dual power supplies and dual network cards. All hard drives use Redundant Arrays of Inexpensive Disk configurations. In addition to dual power supplies, all machines are connected to an uninterrupted power supply that is capable of sending an e-mail alert to the RODS administrator when the main power is down.'), Cord19FullTextSection(title='Health System Resident Component ::: Technical Description', text='An important component of RODS that currently is used only at the UPMC Health System in Pittsburgh is the Health System Resident Component (HSRC). The HSRC is located within the firewall of a health system and connects directly to the HL7 message router. The HSRC currently receives a diverse set of clinical data from the HL7 message router including culture results, radiology reports, and dictated emergency room notes. Its purpose is to provide additional public health surveillance functions that would not be possible if it were located outside of the firewall due to restrictions on the release of identifiable clinical data. The HSRC uses patient identifiers to link laboratory and radiology information to perform case detection. In the past, we have used HSRC to monitor for patients with both a gram-positive rod in a preliminary microbiology culture report and “mediastinal widening” in a radiology report. The HSRC is a case detector in a distributed outbreak detection system that is capable of achieving much higher specificity of patient diagnostic categorization through access to more information.'), Cord19FullTextSection(title='Health System Resident Component ::: Technical Description', text="HSRC also removes identifiable information before transmitting data to the RODS system, a function provided by the health system's message router in other hospitals that connect to RODS."), Cord19FullTextSection(title='Health System Resident Component ::: Technical Description', text='The HSRC at UPMC Health System functions as an electronic laboratory reporting system, although the state and local health departments are not yet ready to receive real-time messaging from the system. Currently, it sends email alerts to the director of the laboratory and hospital infection control group about positive cultures for organisms that are required to be reported to public health in the state of Pennsylvania.30 It also sends messages to hospital infection control when it detects organisms that cause nosocomial infections. These organisms include Clostridium difficile, methicillin-resistant Staphylococcus aureus, and vancomycin-resistant Enterococcus.'), Cord19FullTextSection(title='Health System Resident Component ::: Technical Description', text='We have been able in HSRC to prototype one additional feature, which is a “look-back” function that facilitates very rapid outbreak investigations by providing access to electronic medical records to public health investigators as shown in ▶. This feature requires a token that can be passed to a hospital information system that can uniquely identify a patient, and the reason we have prototyped this feature in the HSRC and not in RODS is simply that HSRC runs within the firewall so an unencrypted token can be used. The look-back is accomplished as follows: when a public health user identifies an anonymous patient record of interest (e.g., one of 20 patients with diarrhea today from one zip code), HSRC calls the UPMC Health System Web-based electronic medical record system and passes it the patient identifier. UPMC Health System then requests the user to log in using the UPMC-issued password before providing access to the record directly from its own secure Web site. This approach is not intended to be implemented in HSRC, but rather in the RODS system outside of the firewall of a health system. It is intended to use encrypted identifiers that the health system would decrypt to retrieve the correct record. The HSRC could provide the encryption-decryption service or it could be provided by another data system in the hospital. We estimate that the prevalence of health systems that have Web-based results review in the United States is 30% to 50% and growing so that this approach could very quickly improve the efficiency of outbreak investigations.  '), Cord19FullTextSection(title='Current Status', text='RODS has been in operation in Pennsylvania since 1999 and in Utah since January 2002. In Utah, RODS receives data from two health systems: Intermountain Heath Care, including nine EDs and 18 acute care facilities, and the University of Utah Health Sciences Center, with one ED.24 Together, these facilities serve about 70% of the population of Utah. In Pennsylvania, RODS receives data from 20 health systems comprising 38 hospitals. Two health systems (each with one hospital) send plain text files to RODS on a daily basis. In Pennsylvania, RODS covers 80% of ED visits in Allegheny County (population, 1.3 M) where Pittsburgh is located; 50% of visits in the 13-county Metropolitan Medical Response Area centered on Pittsburgh (population 3.0 M); and more than 70% coverage of three other counties, including Dauphin County where Harrisburg, the capital of Pennsylvania, is located. The Commonwealth of Pennsylvania is funding a large project to connect the remaining hospitals in the Commonwealth with RODS over the next two years (approximately an additional 170 hospitals).'), Cord19FullTextSection(title='Current Status', text='In December 2002, the RODS laboratory released version 1.1 of the RODS software to the public. The release includes all of the components necessary to deploy RODS for clinic visits surveillance. RODS is free for noncommercial use and can be downloaded at <http://www.health.pitt.edu/rods/sw/>. Although the software has been downloaded in excess of 170 times, we are aware of only a few successful efforts at deployment. These kinds of systems require network engineers, Oracle database administrators, and interface engineers, and very few health departments have access to that skills set.'), Cord19FullTextSection(title='Current Status', text='For these reasons, we have moved to an application service provider model for dissemination in which we encourage state and local health departments to form coalitions to support shared services. We also have been fortunate to have sufficient grant funding from the Commonwealth of Pennsylvania to be able to support these services on an interim basis while sustainable funding models evolve.'), Cord19FullTextSection(title='Discussion', text='Our original design objectives for RODS were real-time collection of data with sufficient geographic coverage and sampling density to provide early syndromic warning of a large-scale aerosol release of anthrax. Although we have not achieved all of our initial design objectives, progress has been substantial. The research identified two types of data—free-text chief complaints and sales of OTC health care products—that can be obtained in real time or near real time at sampling levels of 70% or higher for most of the United States. These results were obtained through large-scale deployments of RODS in Pennsylvania and Utah and through building the National Retail Data Monitor described in the accompanying article in this issue of JAMIA. The deployments also provided insights about organizational and technical success factors that would inform an effort to scale the project nationally.'), Cord19FullTextSection(title='Discussion', text='The project established the importance of HL7 message routers (also known as integration engines) for public health surveillance. HL7 message routers are a mature, highly prevalent technology in health care. We demonstrated that free-text triage chief complaints can be obtained in real time from most U.S. hospitals through message routers and that these data represent early syndromal information about disease. Many other clinical data of value to public health are transmitted using the HL7 standard (e.g., orders for diagnostic tests, especially microbiology tests, reports of chest radiographs, medications, and test results) and can be integrated into RODS or other surveillance systems capable of receiving HL7 messages.'), Cord19FullTextSection(title='Discussion', text='As a result of our efforts to disseminate this technology by giving it away, we have learned that most health departments do not have the technical resources to build and maintain real-time electronic disease surveillance systems. Our application service provider model has been much more successful, and we now recommend that states form coalitions to share the costs of such services.'), Cord19FullTextSection(title='Discussion', text="The project very early identified the need for a computing component to reside within the firewall of a health system, connected to the hospital's HL7 message router. This component would function as a case detector in a distributed public health surveillance scheme linking laboratory and radiology data to increase the specificity of case detection. It has proven very difficult to disseminate this technology, perhaps due to the complexity of the idea. Nevertheless, the threat of bioterrorism has created a need for such technology, and this approach, or something with equivalent function, must be deployed."), Cord19FullTextSection(title='Discussion', text="Adherence to NEDSS architectural standards was an early design objective that we have met. RODS 1.5 closely follows NEDSS architectural, software, messaging, and data specifications. Our success is a strong validation of those standards. We will gain further understanding of the standards as we attempt to use RODS components including HL7 listeners, natural language parsers, message parsers, databases, user interfaces, notification subsystems, and detection algorithms with other NEDSS compliant systems. An ongoing project will use RODS to collect chief complaints and integrate them into the Utah Department of Health's planned NEDSS system."), Cord19FullTextSection(title='Discussion', text='We have demonstrated the ability to rapidly deploy RODS in a special event with the added advantage that the system persisted after the event. This experience suggests strongly that RODS or similar systems be considered an alternative to drop-in surveillance.'), Cord19FullTextSection(title='Discussion', text='Our future plans are to meet our initial design objective to develop early-warning capability for a large, outdoor release of anthrax, especially ensuring that the data and analysis produced by RODS are reviewed by public health. This goal will require improvements in the interfaces and the detection algorithms to reduce false alarms and to vastly improve the efficiency with which anomalies are evaluated by use of multiple types of data, better interfaces, and implementation of the look-back function. We would like to enlarge as quickly as possible the application service provider to include more states and more types of clinical data so that states will be in a position to prospectively evaluate the detection performance from different types of data on naturally occurring outbreaks.'), Cord19FullTextSection(title='Discussion', text='Our long-term goals are to add additional disease scenarios to the design objectives such as detection of in-building anthrax release, vector-borne disease, food-borne disease, and a communicable disease such as severe acute respiratory syndrome (SARS).'), Cord19FullTextSection(title='Conclusion', text='RODS is a NEDSS-compliant public health surveillance system that focuses on real-time collection and analysis of data routinely collected for other purposes. RODS is deployed in two states and was installed quickly in seven weeks for the 2002 Olympics. Our experience demonstrates the feasibility of such a surveillance system and the challenges involved.'), Cord19FullTextSection(title='Conclusion', text='Outbreaks, emerging infections, and bioterrorism have become serious threats. It is our hope that the front-line of public health workers, astute citizens, and health care workers will detect outbreaks early enough so that systems such as RODS are not needed. However, timely outbreak detection is too important to be left to human detection alone. The notion that public health can operate optimally without timely electronic information is as unwise as having commercial airline pilots taking off without weather forecasts and radar.'))),
            192508: Cord19FullTextDoc('pnl9th2c', 'Vascular Life during the COVID-19 Pandemic Reminds Us to Prepare for the Unexpected', '10.1016/j.ejvs.2020.04.040', '2020-05-12', '', (Cord19FullTextSection(title='', text='The first reported case in Spain was on 31 January, and since then the rapid spread of the virus has been demonstrated by the numbers of confirmed diagnoses (> 188 000), and deaths (nearly 20 000) which have increased dramatically every day.'), Cord19FullTextSection(title='', text="The vascular patient is at a higher risk of developing a severe form of the disease due to the nature of its association with several comorbid states and thus, vascular surgery communities from many countries have tried to stratify patients into those requiring urgent care, such as > 70 mm abdominal aortic aneurysm (AAA) or ruptured AAA, critical limb ischaemia (CLI), and symptomatic carotid disease. However, the setting is very dynamic with a clear trend to worsening. Today's circumstances do not fit any previous protocol guidance. An overwhelmed health system, with a lack of personal protection equipment (PPE) and ventilatory support, has led to a global sanitary collapse; with a 90% in hospital occupancy of COVID-19 disease alone, one might correctly guess that the response to remaining diseases, including all cardiovascular ones, is underdiagnosis and undertreatment."), Cord19FullTextSection(title='', text='Moreover, patients are quite afraid (understandably) of coming to hospital, resulting in an unprecedented and unpredictable scenario where surgical arterial practice has severely dropped to an historic minimum: reaching, in total, between 15% and 20% of the usual numbers, mostly related to vascular emergencies (mainly CLI). All ambulatory consultations and imaging follow up (duplex and computed tomography scans) have been rescheduled and consultation by telephone is provided for those awaiting any arterial surgical repair.'), Cord19FullTextSection(title='', text='In addition, for those arriving with any medical or surgical pathology, "war like" decisions are driven by insufficient support to take care of everybody and, therefore, breaking the state of wellbeing that has been largely achieved in most European countries.'), Cord19FullTextSection(title='', text='Therefore, we want to focus on four points that we have faced so far and might help in improving care of, particularly, vascular patients.'), Cord19FullTextSection(title='', text='First, although the aim of protocols is to provide the best treatment for specific situations, they are based mostly on patient assessment. Despite the extensive use of telematic consultation with the aim of identifying patients at risk of clinical worsening (i.e. amputation risk in CLI patients), these "assessment based protocols" are far from working in this unusual scenario. People are simply not coming to hospital as they are extremely afraid; in addition, the first line of disease identification (family medicine) is nearly absent, as family doctors have already been distributed to emergency departments to aid in pulmonary care. Thus, a change in current protocol design considering these novel issues should be created.'), Cord19FullTextSection(title='', text='Second, nearly 20% of COVID-19 positive people are healthcare workers. This is of extreme importance, 3 as the lack of an unaffected workforce could be avoided with adequate and fast COVID testing, 4 receiving specific disease education and by using PPE (unfortunately not adequate in Spain). Moreover, the health professional (potentially infected and working actively with patients) can maintain the negative circle of contagion for those coming to hospital with other kinds of disease and their coworkers. This may lead to the organisation of specific COVID free surgical teams; however, this surely deserves a more in depth discussion in each centre.'), Cord19FullTextSection(title='', text='Third, due to the "extreme" situation, we are forced to optimise surgical care.'), Cord19FullTextSection(title='', text='What does this mean? Not necessarily provide the best care. It actually means "… keep in mind there is no mechanical post-operative support nor ICU beds for your patient …". One might think that an "endovascular first approach" is the best solution to deal with this issue. Protocols suggest treating ruptured AAA by endovascular aneurysm repair if anatomically feasible in patients with long life expectancy. The same may be considered for CLI. However, one may consider the global scenario where all has changed: we are not working in the usual operating room (it has been converted into an ICU for COVID-19 patients); we are without our usual nursing staff (they have been distributed according to hospital needs); and room availability is pretty scarce (as our > 800 bed hospital now shares only four or five COVID free surgical rooms for all surgical specialities plus one for identified COVID-19 patients).'), Cord19FullTextSection(title='', text='One case representing such a scenario is described in Fig. 1 . It is really hard to decide whether an endovascular or open approach seems best for our patients.'), Cord19FullTextSection(title='', text='Moreover, COVID patients are more prone to post-operative complications, as some of them present with lymphopenia, which reflects their immunodeficiency state. 5 We hypothesise that an endovascular first approach (counterbalancing the global aforementioned picture) might be considered, as every patient hospitalised for any type of care should be considered potentially infected and therefore, the fewer wounds and grafts, the less chance of infection.'), Cord19FullTextSection(title='', text='It seems that COVID-19 might present with a procoagulant state (mostly in the inflammatory response and some days after it), as we have observed a higher frequency of deep vein and arterial thrombosis (very much like antiphospholipid syndrome).'), Cord19FullTextSection(title='', text='Further data are required to reach stronger conclusions and, certainly to provide any recommendation for aggressive anticoagulation therapy.'), Cord19FullTextSection(title='', text="We encourage the vascular community to spread any new knowledge related to this new disease. We do not have the answer yet for the best organisation or strategy and therefore, it should be adapted for the hospital/centre's resources. There is a lot of uncertainty now, but also in the future when the COVID-19 pandemic is overcome. Our weakened health system will need to deal with a scenario that might again exceed the available resources. Two weeks later, the patient presented with graft infection and COVID symptoms; he is receiving specific treatment and waiting for explantation and new revascularisation."), Cord19FullTextSection(title='Figure 1', text='A, anterior; and B, lateral view.'))),
        })
        self._test_docs('cord19/trec-covid/round1', count=51078, items={
            0: Cord19Doc('xqhn0vbp', 'Airborne rhinovirus detection and effect of ultraviolet irradiation on detection by a semi-nested RT-PCR assay', '10.1186/1471-2458-3-5', '2003-01-13', re.compile('^BACKGROUND: Rhinovirus, the most common cause of upper respiratory tract infections, has been implic.{1492}ools\\. This method, however, cannot distinguish UV inactivated virus from infectious viral particles\\.$', flags=48)),
            9: Cord19Doc('1ul8owic', "Air pollution and case fatality of SARS in the People's Republic of China: an ecologic study", '10.1186/1476-069x-2-15', '2003-11-20', re.compile('^BACKGROUND: Severe acute respiratory syndrome \\(SARS\\) has claimed 349 lives with 5,327 probable cases.{1763}etrimental effect of air pollution on the prognosis of SARS patients deserves further investigation\\.$', flags=48)),
            51077: Cord19Doc('1peoakxt', 'The pathogenic role of virus-specific antibody secreting cells in the CNS of rats resistant and susceptible to coronavirus-induced encephalitis', '10.1016/0165-5728(91)91228-5', '1991-12-31', ''),
        })
        self._test_docs('cord19/trec-covid/round2', count=59887, items={
            0: Cord19Doc('zjufx4fo', 'Sequence requirements for RNA strand transfer during nidovirus discontinuous subgenomic RNA synthesis', '10.1093/emboj/20.24.7220', '2001-12-17', re.compile('^Nidovirus subgenomic mRNAs contain a leader sequence derived from the 5′ end of the genome fused to .{968} of TRS mutants strongly suggested that the discontinuous step occurs during minus strand synthesis\\.$', flags=48)),
            9: Cord19Doc('3oxzzxnd', 'SseG, a virulence protein that targets Salmonella to the Golgi network', '10.1093/emboj/cdg517', '2003-10-01', re.compile('^Intracellular replication of the bacterial pathogen Salmonella enterica occurs in membrane\\-bound com.{1028} is dependent on simultaneous and selective interactions with both endocytic and secretory pathways\\.$', flags=48)),
            59886: Cord19Doc('jxbr6hbk', 'Identifying spatio-temporal dynamics of Ebola in Sierra Leone using virus genomes', '10.1098/rsif.2017.0583', '2017-11-29', re.compile('^Containing the recent West African outbreak of Ebola virus \\(EBOV\\) required the deployment of substan.{1158}ffering operationally relevant guidance for surveillance and sampling strategies during an epidemic\\.$', flags=48)),
        })
        self._test_docs('cord19/trec-covid/round3', count=128492, items={
            0: Cord19Doc('ug7v899j', 'Clinical features of culture-proven Mycoplasma pneumoniae infections at King Abdulaziz University Hospital, Jeddah, Saudi Arabia', '10.1186/1471-2334-1-6', '2001-07-04', re.compile('^OBJECTIVE: This retrospective chart review describes the epidemiology and clinical features of 40 pa.{1647}preschool children and that the mortality rate of pneumonia in patients with comorbidities was high\\.$', flags=48)),
            9: Cord19Doc('i0zym7iq', 'Discontinuous and non-discontinuous subgenomic RNA transcription in a nidovirus', '10.1093/emboj/cdf635', '2002-12-01', re.compile('^Arteri\\-, corona\\-, toro\\- and roniviruses are evolutionarily related positive\\-strand RNA viruses, unit.{1021}ortant implications for the understanding of the mechanism and evolution of nidovirus transcription\\.$', flags=48)),
            128491: Cord19Doc('clmtwq4v', 'The determinants of the 1999 and 2007 Chinese Golden Holiday System: A content analysis of official documentation', '10.1016/j.tourman.2009.10.003', '2009-11-06', re.compile('^This study examines the factors that lay behind the development of the Golden Week holiday system in.{587} in 2007\\. The theoretical contributions and practical implications of this study are also addressed\\.$', flags=48)),
        })
        self._test_docs('cord19/trec-covid/round4', count=158274, items={
            0: Cord19Doc('ug7v899j', 'Clinical features of culture-proven Mycoplasma pneumoniae infections at King Abdulaziz University Hospital, Jeddah, Saudi Arabia', '10.1186/1471-2334-1-6', '2001-07-04', re.compile('^OBJECTIVE: This retrospective chart review describes the epidemiology and clinical features of 40 pa.{1647}preschool children and that the mortality rate of pneumonia in patients with comorbidities was high\\.$', flags=48)),
            9: Cord19Doc('jg13scgo', 'Technical Description of RODS: A Real-time Public Health Surveillance System', '10.1197/jamia.m1345', '2003-09-01', re.compile('^This report describes the design and implementation of the Real\\-time Outbreak and Disease Surveillan.{1077} be a resource for implementing, evaluating, and applying new methods of public health surveillance\\.$', flags=48)),
            158273: Cord19Doc('zvop8bxh', 'Antiviral RNAi therapy: emerging approaches for hitting a moving target', '10.1038/sj.gt.3302645', '2005-09-22', re.compile('^The field of directed RNA interference \\(RNAi\\) has rapidly developed into a highly promising approach.{1184}NAi targets and discuss strategies for translating these findings into effective clinical therapies\\.$', flags=48)),
        })
        self._test_docs('cord19/trec-covid/round5', count=192509, items={
            0: Cord19Doc('ug7v899j', 'Clinical features of culture-proven Mycoplasma pneumoniae infections at King Abdulaziz University Hospital, Jeddah, Saudi Arabia', '10.1186/1471-2334-1-6', '2001-07-04', re.compile('^OBJECTIVE: This retrospective chart review describes the epidemiology and clinical features of 40 pa.{1647}preschool children and that the mortality rate of pneumonia in patients with comorbidities was high\\.$', flags=48)),
            9: Cord19Doc('jg13scgo', 'Technical Description of RODS: A Real-time Public Health Surveillance System', '10.1197/jamia.m1345', '2003-09-01', re.compile('^This report describes the design and implementation of the Real\\-time Outbreak and Disease Surveillan.{1077} be a resource for implementing, evaluating, and applying new methods of public health surveillance\\.$', flags=48)),
            192508: Cord19Doc('pnl9th2c', 'Vascular Life during the COVID-19 Pandemic Reminds Us to Prepare for the Unexpected', '10.1016/j.ejvs.2020.04.040', '2020-05-12', ''),
        })

    def test_cord19_queries(self):
        self._test_queries('cord19/trec-covid', count=50, items={
            0: TrecQuery('1', 'coronavirus origin', 'what is the origin of COVID-19', "seeking range of information about the SARS-CoV-2 virus's origin, including its evolution, animal source, and first transmission into humans"),
            9: TrecQuery('10', 'coronavirus social distancing impact', 'has social distancing had an impact on slowing the spread of COVID-19?', "seeking specific information on studies that have measured COVID-19's transmission in one or more social distancing (or non-social distancing) approaches"),
            49: TrecQuery('50', 'mRNA vaccine coronavirus', 'what is known about an mRNA vaccine for the SARS-CoV-2 virus?', 'Looking for studies specifically focusing on mRNA vaccines for COVID-19, including how mRNA vaccines work, why they are promising, and any results from actual clinical studies.'),
        })
        self._test_queries('cord19/fulltext/trec-covid', count=50, items={
            0: TrecQuery('1', 'coronavirus origin', 'what is the origin of COVID-19', "seeking range of information about the SARS-CoV-2 virus's origin, including its evolution, animal source, and first transmission into humans"),
            9: TrecQuery('10', 'coronavirus social distancing impact', 'has social distancing had an impact on slowing the spread of COVID-19?', "seeking specific information on studies that have measured COVID-19's transmission in one or more social distancing (or non-social distancing) approaches"),
            49: TrecQuery('50', 'mRNA vaccine coronavirus', 'what is known about an mRNA vaccine for the SARS-CoV-2 virus?', 'Looking for studies specifically focusing on mRNA vaccines for COVID-19, including how mRNA vaccines work, why they are promising, and any results from actual clinical studies.'),
        })
        self._test_queries('cord19/trec-covid/round1', count=30, items={
            0: TrecQuery('1', 'coronavirus origin', 'what is the origin of COVID-19', "seeking range of information about the SARS-CoV-2 virus's origin, including its evolution, animal source, and first transmission into humans"),
            9: TrecQuery('10', 'coronavirus social distancing impact', 'has social distancing had an impact on slowing the spread of COVID-19?', "seeking specific information on studies that have measured COVID-19's transmission in one or more social distancing (or non-social distancing) approaches"),
            29: TrecQuery('30', 'coronavirus remdesivir', 'is remdesivir an effective treatment for COVID-19', 'seeking specific information on clinical outcomes in COVID-19 patients treated with remdesivir'),
        })
        self._test_queries('cord19/trec-covid/round2', count=35, items={
            0: TrecQuery('1', 'coronavirus origin', 'what is the origin of COVID-19', "seeking range of information about the SARS-CoV-2 virus's origin, including its evolution, animal source, and first transmission into humans"),
            9: TrecQuery('10', 'coronavirus social distancing impact', 'has social distancing had an impact on slowing the spread of COVID-19?', "seeking specific information on studies that have measured COVID-19's transmission in one or more social distancing (or non-social distancing) approaches"),
            34: TrecQuery('35', 'coronavirus public datasets', 'What new public datasets are available related to COVID-19?', 'Seeking articles that specifically release new data related to SARS-CoV-2 or COVID-19, including genomic data, patient data, public health data, etc. Articles that reference previously existing datasets are not relevant.'),
        })
        self._test_queries('cord19/trec-covid/round3', count=40, items={
            0: TrecQuery('1', 'coronavirus origin', 'what is the origin of COVID-19', "seeking range of information about the SARS-CoV-2 virus's origin, including its evolution, animal source, and first transmission into humans"),
            9: TrecQuery('10', 'coronavirus social distancing impact', 'has social distancing had an impact on slowing the spread of COVID-19?', "seeking specific information on studies that have measured COVID-19's transmission in one or more social distancing (or non-social distancing) approaches"),
            39: TrecQuery('40', 'coronavirus mutations', 'What are the observed mutations in the SARS-CoV-2 genome and how often do the mutations occur?', 'Looking for studies that describes the emergence of genomic diversity of the coronavirus due to recurrent mutations which explore the potential genomic site of the mutation, mechanisms and its potential or observed clinical implications in the pathogenicity of the virus'),
        })
        self._test_queries('cord19/trec-covid/round4', count=45, items={
            0: TrecQuery('1', 'coronavirus origin', 'what is the origin of COVID-19', "seeking range of information about the SARS-CoV-2 virus's origin, including its evolution, animal source, and first transmission into humans"),
            9: TrecQuery('10', 'coronavirus social distancing impact', 'has social distancing had an impact on slowing the spread of COVID-19?', "seeking specific information on studies that have measured COVID-19's transmission in one or more social distancing (or non-social distancing) approaches"),
            44: TrecQuery('45', 'coronavirus mental health impact', 'How has the COVID-19 pandemic impacted mental health?', 'Includes increasing/decreasing rates of depression, anxiety, panic disorder, and other psychiatric and mental health conditions.'),
        })
        self._test_queries('cord19/trec-covid/round5', count=50, items={
            0: TrecQuery('1', 'coronavirus origin', 'what is the origin of COVID-19', "seeking range of information about the SARS-CoV-2 virus's origin, including its evolution, animal source, and first transmission into humans"),
            9: TrecQuery('10', 'coronavirus social distancing impact', 'has social distancing had an impact on slowing the spread of COVID-19?', "seeking specific information on studies that have measured COVID-19's transmission in one or more social distancing (or non-social distancing) approaches"),
            49: TrecQuery('50', 'mRNA vaccine coronavirus', 'what is known about an mRNA vaccine for the SARS-CoV-2 virus?', 'Looking for studies specifically focusing on mRNA vaccines for COVID-19, including how mRNA vaccines work, why they are promising, and any results from actual clinical studies.'),
        })

    def test_cord19_qrels(self):
        self._test_qrels('cord19/trec-covid', count=69318, items={
            0: TrecQrel('1', '005b2j4b', 2, '4.5'),
            9: TrecQrel('1', '05vx82oo', 0, '3'),
            69317: TrecQrel('50', 'zz8wvos9', 1, '5'),
        })
        self._test_qrels('cord19/fulltext/trec-covid', count=69318, items={
            0: TrecQrel('1', '005b2j4b', 2, '4.5'),
            9: TrecQrel('1', '05vx82oo', 0, '3'),
            69317: TrecQrel('50', 'zz8wvos9', 1, '5'),
        })
        self._test_qrels('cord19/trec-covid/round1', count=8691, items={
            0: TrecQrel('1', '010vptx3', 2, '0.5'),
            9: TrecQrel('1', '0pbjttv4', 0, '1'),
            8690: TrecQrel('30', 'zn87f1lk', 1, '0.5'),
        })
        self._test_qrels('cord19/trec-covid/round2', count=12037, items={
            0: TrecQrel('1', '08efpohc', 0, '1.5'),
            9: TrecQrel('1', '1bvsn9e8', 2, '2'),
            12036: TrecQrel('35', 'zzmfhr2s', 0, '2'),
        })
        self._test_qrels('cord19/trec-covid/round3', count=12713, items={
            0: TrecQrel('1', '0194oljo', 1, '2.5'),
            9: TrecQrel('1', '0qpfoh5t', 0, '3'),
            12712: TrecQrel('40', 'zsx7wfyj', 1, '3'),
        })
        self._test_qrels('cord19/trec-covid/round4', count=13262, items={
            0: TrecQrel('1', '00fmeepz', 1, '4'),
            9: TrecQrel('1', '1c47w4q5', 2, '4'),
            13261: TrecQrel('45', 'zzrsk1ls', 2, '4'),
        })
        self._test_qrels('cord19/trec-covid/round5', count=23151, items={
            0: TrecQrel('1', '005b2j4b', 2, '4.5'),
            9: TrecQrel('1', '1ag9jkk6', 0, '4.5'),
            23150: TrecQrel('50', 'zz8wvos9', 1, '5'),
        })


if __name__ == '__main__':
    unittest.main()
