
@article{raffel_exploring_2020,
	title = {Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer},
	url = {http://arxiv.org/abs/1910.10683},
	abstract = {Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing ({NLP}). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for {NLP} by introducing a unified framework that converts all text-based language problems into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled data sets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new ``Colossal Clean Crawled Corpus'', we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for {NLP}, we release our data set, pre-trained models, and code.},
	journaltitle = {{arXiv}:1910.10683 [cs, stat]},
	author = {Raffel, Colin and Shazeer, Noam and Roberts, Adam and Lee, Katherine and Narang, Sharan and Matena, Michael and Zhou, Yanqi and Li, Wei and Liu, Peter J.},
	urldate = {2021-02-01},
	date = {2020-07-28},
	eprinttype = {arxiv},
	eprint = {1910.10683},
	keywords = {Computer Science - Computation and Language, Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/au554730/Zotero/storage/QABZI5PK/Raffel et al. - 2020 - Exploring the Limits of Transfer Learning with a U.pdf:application/pdf;arXiv.org Snapshot:/Users/au554730/Zotero/storage/T37AKL3B/1910.html:text/html},
}

@report{rae_scaling_2022,
	title = {Scaling Language Models: Methods, Analysis \& Insights from Training Gopher},
	url = {http://arxiv.org/abs/2112.11446},
	shorttitle = {Scaling Language Models},
	abstract = {Language modelling provides a step towards intelligent communication systems by harnessing large repositories of written human knowledge to better predict and understand the world. In this paper, we present an analysis of Transformer-based language model performance across a wide range of model scales -- from models with tens of millions of parameters up to a 280 billion parameter model called Gopher. These models are evaluated on 152 diverse tasks, achieving state-of-the-art performance across the majority. Gains from scale are largest in areas such as reading comprehension, fact-checking, and the identification of toxic language, but logical and mathematical reasoning see less benefit. We provide a holistic analysis of the training dataset and model's behaviour, covering the intersection of model scale with bias and toxicity. Finally we discuss the application of language models to {AI} safety and the mitigation of downstream harms.},
	number = {{arXiv}:2112.11446},
	institution = {{arXiv}},
	author = {Rae, Jack W. and Borgeaud, Sebastian and Cai, Trevor and Millican, Katie and Hoffmann, Jordan and Song, Francis and Aslanides, John and Henderson, Sarah and Ring, Roman and Young, Susannah and Rutherford, Eliza and Hennigan, Tom and Menick, Jacob and Cassirer, Albin and Powell, Richard and Driessche, George van den and Hendricks, Lisa Anne and Rauh, Maribeth and Huang, Po-Sen and Glaese, Amelia and Welbl, Johannes and Dathathri, Sumanth and Huang, Saffron and Uesato, Jonathan and Mellor, John and Higgins, Irina and Creswell, Antonia and {McAleese}, Nat and Wu, Amy and Elsen, Erich and Jayakumar, Siddhant and Buchatskaya, Elena and Budden, David and Sutherland, Esme and Simonyan, Karen and Paganini, Michela and Sifre, Laurent and Martens, Lena and Li, Xiang Lorraine and Kuncoro, Adhiguna and Nematzadeh, Aida and Gribovskaya, Elena and Donato, Domenic and Lazaridou, Angeliki and Mensch, Arthur and Lespiau, Jean-Baptiste and Tsimpoukelli, Maria and Grigorev, Nikolai and Fritz, Doug and Sottiaux, Thibault and Pajarskas, Mantas and Pohlen, Toby and Gong, Zhitao and Toyama, Daniel and d'Autume, Cyprien de Masson and Li, Yujia and Terzi, Tayfun and Mikulik, Vladimir and Babuschkin, Igor and Clark, Aidan and Casas, Diego de Las and Guy, Aurelia and Jones, Chris and Bradbury, James and Johnson, Matthew and Hechtman, Blake and Weidinger, Laura and Gabriel, Iason and Isaac, William and Lockhart, Ed and Osindero, Simon and Rimell, Laura and Dyer, Chris and Vinyals, Oriol and Ayoub, Kareem and Stanway, Jeff and Bennett, Lorrayne and Hassabis, Demis and Kavukcuoglu, Koray and Irving, Geoffrey},
	urldate = {2022-06-03},
	date = {2022-01-21},
	doi = {10.48550/arXiv.2112.11446},
	eprinttype = {arxiv},
	eprint = {2112.11446 [cs]},
	note = {type: article},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language},
	file = {arXiv.org Snapshot:/Users/au554730/Zotero/storage/KNAZNFHY/2112.html:text/html;Rae et al_2022_Scaling Language Models.pdf:/Users/au554730/Library/Mobile Documents/com~apple~CloudDocs/zotero_storage/Rae et al_2022_Scaling Language Models.pdf:application/pdf},
}

@article{hansen_lexical_2022,
	title = {Lexical Stability of Psychiatric Clinical Notes from Electronic Health Records over a Decade},
	url = {http://medrxiv.org/content/early/2022/09/06/2022.09.05.22279610.abstract},
	doi = {10.1101/2022.09.05.22279610},
	abstract = {Natural Language Processing methods hold promise for improving clinical prediction by utilising information otherwise hidden in the clinical notes of electronic health records. However, clinical practice—as well as the systems and databases in which clinical notes are recorded and stored—change over time. As a consequence, the content of clinical notes may also change over time, which could degrade the performance of prediction models. Despite its importance, the stability of clinical notes over time has rarely been tested. Therefore, in this study, we examined the lexical stability of clinical notes from the Psychiatric Services of the Central Denmark Region in the period from January 1, 2011, to November 22, 2021 (a total of 14,811,551 clinical notes describing 129,570 patients) by quantifying sentence length, readability, syntactic complexity and clinical content - and estimating changepoints in these metrics. We find lexical and syntactic stability over time, which bodes well for the use of Natural Language Processing for predictive modelling in clinical practice.Competing Interest {StatementDanielsen} has received a speaker honorarium from Otsuka Pharmaceutical. {SDØ} received the 2020 Lundbeck Foundation Young Investigator Prize. Furthermore, {SDØ} owns/has owned units of mutual funds with stock tickers {DKIGI}, {IAIMWC} and {WEKAFKI}, and has owned units of exchange-traded funds with stock tickers {BATE}, {TRET}, {QDV}5, {QDVH}, {QDVE}, {SADM}, {IQQH}, {USPY}, {EXH}2, 2B76 and {EUNL}. The remaining authors declare no conflicts of interest.Funding {StatementThe} study is supported by grants from the Lundbeck Foundation (grant number: R344-2020- 1073), the Danish Cancer Society (grant number: R283-A16461), the Central Denmark Region Fund for Strengthening of Health Science (grant number: 1-36-72-4-20) and the Danish Agency for Digitisation Investment Fund for New Technologies (grant number 2020- 6720) to Østergaard, who reports further funding from the Lundbeck Foundation (grant number: R358-2020-2341), the Novo Nordisk Foundation (grant number: {NNF}20SA0062874) and Independent Research Fund Denmark (grant number: 7016-00048B). The funders played no role in the study design, collection, analysis or interpretation of data, the writing of the report or the decision to submit the paper for publication.Author {DeclarationsI} confirm all relevant ethical guidelines have been followed, and any necessary {IRB} and/or ethics committee approvals have been obtained.{YesThe} details of the {IRB}/oversight body that provided approval or exemption for the research described are given below:The Central Denmark Region Legal office gave permission to use electronic heatlh records. According to the Danish Committee Act, ethical review board approval is not required for studies based solely on data from electronic health records (waiver for this project: 1-10-72-1-22).I confirm that all necessary patient/participant consent has been obtained and the appropriate institutional forms have been archived, and that any patient/participant/sample identifiers included were not known to anyone (e.g., hospital staff, patients or participants themselves) outside the research group so cannot be used to identify individuals.{YesI} understand that all clinical trials and any other prospective interventional studies must be registered with an {ICMJE}-approved registry, such as {ClinicalTrials}.gov. I confirm that any such study reported in the manuscript has been registered and the trial registration {ID} is provided (note: if posting a prospective study registered retrospectively, please provide a statement in the trial {ID} field explaining why the study was not registered in advance).{YesI} have followed all appropriate research reporting guidelines and uploaded the relevant {EQUATOR} Network research reporting checklist(s) and other pertinent material as supplementary files, if applicable.{YesThe} data used for the study is not publicly available as it contains personally sensitive information. All code can be found online at https://github.com/Aarhus-Psychiatry-Research/lexical-stability https://github.com/Aarhus-Psychiatry-Research/lexical-stability},
	pages = {2022.09.05.22279610},
	journaltitle = {{medRxiv}},
	author = {Hansen, Lasse and Enevoldsen, Kenneth and Bernstorff, Martin and Perfalk, Erik and Danielsen, Andreas A. and Nielbo, Kristoffer L. and Østergaard, Søren D.},
	date = {2022-01-01},
}

@software{honnibal_spacy_2020,
	title = {{spaCy}: Industrial-strength Natural Language Processing in Python},
	rights = {{MIT}},
	url = {https://github.com/explosion/spaCy/blob/abb0ab109d33d2deaa6155a61fad649a25472f9c/CITATION.cff},
	shorttitle = {{spaCy}},
	abstract = {Industrial-strength Natural Language Processing ({NLP}) in Python},
	author = {Honnibal, Matthew and Montani, Ines and Van Landeghem, Sofie and Boyd, Adriane},
	urldate = {2023-01-02},
	date = {2020},
	doi = {10.5281/zenodo.1212303},
	note = {original-date: 2014-07-03T15:15:40Z},
}

@thesis{tannert_skriftsproglig_2023,
	title = {Skriftsproglig udvikling i grundskolens danskfag},
	institution = {Aarhus University},
	type = {phdthesis},
	author = {Tannert, Morten},
	date = {2023},
}

@software{ward_textstat_2022,
	title = {Textstat},
	rights = {{MIT}},
	url = {https://github.com/textstat/textstat},
	abstract = {python package to calculate readability statistics of a text object - paragraphs, sentences, articles.},
	publisher = {Textstat},
	author = {Ward, Alex},
	urldate = {2023-01-02},
	date = {2022-12-31},
	note = {original-date: 2014-06-18T10:54:08Z},
	keywords = {flesch-kincaid-grade, flesch-reading-ease, python, readability, smog, textstat},
}

@software{holtzscher_spacy-readability_2019,
	title = {spacy-readability: {spaCy} pipeline component for adding text readability meta data to Doc objects.},
	rights = {{MIT} License},
	shorttitle = {spacy-readability},
	version = {1.4.1},
	author = {Holtzscher, Michael},
	date = {2019},
	file = {Snapshot:/Users/au554730/Zotero/storage/XB2VGBVP/spacy-readability.html:text/html},
}

@software{dewilde_textacy_2021,
	title = {textacy: {NLP}, before and after {spaCy}},
	rights = {Apache Software License},
	url = {https://github.com/chartbeat-labs/textacy},
	shorttitle = {textacy},
	version = {0.12.0},
	author = {{DeWilde}, Burton},
	urldate = {2023-01-02},
	date = {2021},
	keywords = {linguistics, nlp,, spacy,, Text Processing - Linguistic, text processing,},
	file = {Snapshot:/Users/au554730/Zotero/storage/F7N9C8AR/textacy.html:text/html},
}

@article{dubay_principles_2004,
	title = {The principles of readability.},
	journaltitle = {Online Submission},
	author = {{DuBay}, William H.},
	date = {2004},
	note = {Publisher: {ERIC}},
	file = {DuBay_2004_The principles of readability.pdf:/Users/au554730/Library/Mobile Documents/com~apple~CloudDocs/zotero_storage/DuBay_2004_The principles of readability2.pdf:application/pdf;Snapshot:/Users/au554730/Zotero/storage/6EIRJLVU/eric.ed.gov.html:text/html},
}

@article{liu_dependency_2008,
	title = {Dependency distance as a metric of language comprehension difficulty},
	volume = {9},
	pages = {159--191},
	number = {2},
	doi = {10.17791/jcs.2008.9.2.159},
	journaltitle = {Journal of Cognitive Science},
	author = {Liu, Haitao},
	date = {2008},
	file = {Liu_2008_Dependency distance as a metric of language comprehension difficulty.pdf:/Users/au554730/Library/Mobile Documents/com~apple~CloudDocs/zotero_storage/Liu_2008_Dependency distance as a metric of language comprehension difficulty.pdf:application/pdf},
}

@article{gibson_how_2019,
	title = {How efficiency shapes human language},
	volume = {23},
	pages = {389--407},
	number = {5},
	journaltitle = {Trends in cognitive sciences},
	author = {Gibson, Edward and Futrell, Richard and Piantadosi, Steven P. and Dautriche, Isabelle and Mahowald, Kyle and Bergen, Leon and Levy, Roger},
	date = {2019},
	note = {Publisher: Elsevier},
	file = {Full Text:/Users/au554730/Zotero/storage/QCUGU8Y5/S1364661319300580.html:text/html},
}

@article{bedi_automated_2015,
	title = {Automated analysis of free speech predicts psychosis onset in high-risk youths},
	volume = {1},
	rights = {2015 The Author(s)},
	issn = {2334-265X},
	url = {https://www.nature.com/articles/npjschz201530},
	doi = {10.1038/npjschz.2015.30},
	abstract = {Psychiatry lacks the objective clinical tests routinely used in other specializations. Novel computerized methods to characterize complex behaviors such as speech could be used to identify and predict psychiatric illness in individuals.},
	pages = {1--7},
	number = {1},
	journaltitle = {npj Schizophrenia},
	shortjournal = {npj Schizophr},
	author = {Bedi, Gillinder and Carrillo, Facundo and Cecchi, Guillermo A. and Slezak, Diego Fernández and Sigman, Mariano and Mota, Natália B. and Ribeiro, Sidarta and Javitt, Daniel C. and Copelli, Mauro and Corcoran, Cheryl M.},
	urldate = {2023-01-03},
	date = {2015-08-26},
	langid = {english},
	note = {Number: 1
Publisher: Nature Publishing Group},
	keywords = {Neuroscience, Schizophrenia},
	file = {Bedi et al_2015_Automated analysis of free speech predicts psychosis onset in high-risk youths.pdf:/Users/au554730/Library/Mobile Documents/com~apple~CloudDocs/zotero_storage/Bedi et al_2015_Automated analysis of free speech predicts psychosis onset in high-risk youths.pdf:application/pdf},
}

@article{parola_speech_2022,
	title = {Speech disturbances in schizophrenia: Assessing cross-linguistic generalizability of {NLP} automated measures of coherence},
	issn = {0920-9964},
	url = {https://www.sciencedirect.com/science/article/pii/S0920996422002742},
	doi = {10.1016/j.schres.2022.07.002},
	shorttitle = {Speech disturbances in schizophrenia},
	abstract = {Introduction
Language disorders – disorganized and incoherent speech in particular - are distinctive features of schizophrenia. Natural language processing ({NLP}) offers automated measures of incoherent speech as promising markers for schizophrenia. However, the scientific and clinical impact of {NLP} markers depends on their generalizability across contexts, samples, and languages, which we systematically assessed in the present study relying on a large, novel, cross-linguistic corpus.
Methods
We collected a Danish ({DK}), German ({GE}), and Chinese ({CH}) cross-linguistic dataset involving transcripts from 187 participants with schizophrenia (111DK, 25GE, 51CH) and 200 matched controls (129DK, 29GE, 42CH) performing the Animated Triangles Task. Fourteen previously published {NLP} coherence measures were calculated, and between-groups differences and association with symptoms were tested for cross-linguistic generalizability.
Results
One coherence measure, i.e. second-order coherence, robustly generalized across samples and languages. We found several language-specific effects, some of which partially replicated previous findings (lower coherence in German and Chinese patients), while others did not (higher coherence in Danish patients). We found several associations between symptoms and measures of coherence, but the effects were generally inconsistent across languages and rating scales.
Conclusions
Using a cumulative approach, we have shown that {NLP} findings of reduced semantic coherence in schizophrenia have limited generalizability across different languages, samples, and measures. We argue that several factors such as sociodemographic and clinical heterogeneity, cross-linguistic variation, and the different {NLP} measures reflecting different clinical aspects may be responsible for this variability. Future studies should take this variability into account in order to develop effective clinical applications targeting different patient populations.},
	journaltitle = {Schizophrenia Research},
	shortjournal = {Schizophrenia Research},
	author = {Parola, Alberto and Lin, Jessica Mary and Simonsen, Arndis and Bliksted, Vibeke and Zhou, Yuan and Wang, Huiling and Inoue, Lana and Koelkebeck, Katja and Fusaroli, Riccardo},
	urldate = {2023-01-03},
	date = {2022-08-01},
	langid = {english},
	keywords = {Biomarker, Communication disorders, Digital phenotyping, Natural language processing, Schizophrenia spectrum disorder, Semantic coherence, Thought disorder},
	file = {Parola et al_2022_Speech disturbances in schizophrenia.pdf:/Users/au554730/Library/Mobile Documents/com~apple~CloudDocs/zotero_storage/Parola et al_2022_Speech disturbances in schizophrenia2.pdf:application/pdf;ScienceDirect Snapshot:/Users/au554730/Zotero/storage/64C5DAP6/S0920996422002742.html:text/html},
}

@article{tang_natural_2021,
	title = {Natural language processing methods are sensitive to sub-clinical linguistic differences in schizophrenia spectrum disorders},
	volume = {7},
	rights = {2021 The Author(s)},
	issn = {2334-265X},
	url = {https://www.nature.com/articles/s41537-021-00154-3},
	doi = {10.1038/s41537-021-00154-3},
	abstract = {Computerized natural language processing ({NLP}) allows for objective and sensitive detection of speech disturbance, a hallmark of schizophrenia spectrum disorders ({SSD}). We explored several methods for characterizing speech changes in {SSD} (n = 20) compared to healthy control ({HC}) participants (n = 11) and approached linguistic phenotyping on three levels: individual words, parts-of-speech ({POS}), and sentence-level coherence. {NLP} features were compared with a clinical gold standard, the Scale for the Assessment of Thought, Language and Communication ({TLC}). We utilized Bidirectional Encoder Representations from Transformers ({BERT}), a state-of-the-art embedding algorithm incorporating bidirectional context. Through the {POS} approach, we found that {SSD} used more pronouns but fewer adverbs, adjectives, and determiners (e.g., “the,” “a,”). Analysis of individual word usage was notable for more frequent use of first-person singular pronouns among individuals with {SSD} and first-person plural pronouns among {HC}. There was a striking increase in incomplete words among {SSD}. Sentence-level analysis using {BERT} reflected increased tangentiality among {SSD} with greater sentence embedding distances. The {SSD} sample had low speech disturbance on average and there was no difference in group means for {TLC} scores. However, {NLP} measures of language disturbance appear to be sensitive to these subclinical differences and showed greater ability to discriminate between {HC} and {SSD} than a model based on clinical ratings alone. These intriguing exploratory results from a small sample prompt further inquiry into {NLP} methods for characterizing language disturbance in {SSD} and suggest that {NLP} measures may yield clinically relevant and informative biomarkers.},
	pages = {1--8},
	number = {1},
	journaltitle = {npj Schizophrenia},
	shortjournal = {npj Schizophr},
	author = {Tang, Sunny X. and Kriz, Reno and Cho, Sunghye and Park, Suh Jung and Harowitz, Jenna and Gur, Raquel E. and Bhati, Mahendra T. and Wolf, Daniel H. and Sedoc, João and Liberman, Mark Y.},
	urldate = {2023-01-03},
	date = {2021-05-14},
	langid = {english},
	note = {Number: 1
Publisher: Nature Publishing Group},
	keywords = {Biomarkers, Psychosis},
	file = {Tang et al_2021_Natural language processing methods are sensitive to sub-clinical linguistic.pdf:/Users/au554730/Library/Mobile Documents/com~apple~CloudDocs/zotero_storage/Tang et al_2021_Natural language processing methods are sensitive to sub-clinical linguistic.pdf:application/pdf},
}

@article{hansen_automated_2023,
	title = {Automated speech- and text-based classification of neuropsychiatric conditions in a multidiagnostic setting},
	url = {http://arxiv.org/abs/2301.06916},
	doi = {10.48550/arXiv.2301.06916},
	abstract = {Speech patterns have been identified as potential diagnostic markers for neuropsychiatric conditions. However, most studies only compare a single clinical group to healthy controls, whereas clinical practice often requires differentiating between multiple potential diagnoses (multiclass settings). To address this, we assembled a dataset of repeated recordings from 420 participants (67 with major depressive disorder, 106 with schizophrenia and 46 with autism, as well as matched controls), and tested the performance of a range of conventional machine learning models and advanced Transformer models on both binary and multiclass classification, based on voice and text features. While binary models performed comparably to previous research (F1 scores between 0.54-0.75 for autism spectrum disorder, {ASD}; 0.67-0.92 for major depressive disorder, {MDD}; and 0.71-0.83 for schizophrenia); when differentiating between multiple diagnostic groups performance decreased markedly (F1 scores between 0.35-0.44 for {ASD}, 0.57-0.75 for {MDD}, 0.15-0.66 for schizophrenia, and 0.38-0.52 macro F1). Combining voice and text-based models yielded increased performance, suggesting that they capture complementary diagnostic information. Our results indicate that models trained on binary classification may learn to rely on markers of generic differences between clinical and non-clinical populations, or markers of clinical features that overlap across conditions, rather than identifying markers specific to individual conditions. We provide recommendations for future research in the field, suggesting increased focus on developing larger transdiagnostic datasets that include more fine-grained clinical features, and that can support the development of models that better capture the complexity of neuropsychiatric conditions and naturalistic diagnostic assessment.},
	number = {{arXiv}:2301.06916},
	publisher = {{arXiv}},
	author = {Hansen, Lasse and Rocca, Roberta and Simonsen, Arndis and Parola, Alberto and Bliksted, Vibeke and Ladegaard, Nicolai and Bang, Dan and Tylén, Kristian and Weed, Ethan and Østergaard, Søren Dinesen and Fusaroli, Riccardo},
	urldate = {2023-02-17},
	date = {2023-01-31},
	eprinttype = {arxiv},
	eprint = {2301.06916 [cs, stat]},
	keywords = {Computer Science - Computation and Language, Computer Science - Machine Learning, Computer Science - Sound, Statistics - Applications},
}
