
@misc{wang_disentangled_2024,
	title = {Disentangled Representation Learning},
	url = {http://arxiv.org/abs/2211.11695},
	doi = {10.48550/arXiv.2211.11695},
	abstract = {Disentangled Representation Learning ({DRL}) aims to learn a model capable of identifying and disentangling the underlying factors hidden in the observable data in representation form. The process of separating underlying factors of variation into variables with semantic meaning benefits in learning explainable representations of data, which imitates the meaningful understanding process of humans when observing an object or relation. As a general learning strategy, {DRL} has demonstrated its power in improving the model explainability, controlability, robustness, as well as generalization capacity in a wide range of scenarios such as computer vision, natural language processing, and data mining. In this article, we comprehensively investigate {DRL} from various aspects including motivations, definitions, methodologies, evaluations, applications, and model designs. We first present two well-recognized definitions, i.e., Intuitive Definition and Group Theory Definition for disentangled representation learning. We further categorize the methodologies for {DRL} into four groups from the following perspectives, the model type, representation structure, supervision signal, and independence assumption. We also analyze principles to design different {DRL} models that may benefit different tasks in practical applications. Finally, we point out challenges in {DRL} as well as potential research directions deserving future investigations. We believe this work may provide insights for promoting the {DRL} research in the community.},
	number = {{arXiv}:2211.11695},
	publisher = {{arXiv}},
	author = {Wang, Xin and Chen, Hong and Tang, Si'ao and Wu, Zihao and Zhu, Wenwu},
	urldate = {2026-03-27},
	date = {2024-06-26},
	eprinttype = {arxiv},
	eprint = {2211.11695 [cs]},
	keywords = {Computer Science - Machine Learning, Computer Science - Artificial Intelligence},
	file = {Preprint PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/DX7LVLCR/Wang et al. - 2024 - Disentangled Representation Learning.pdf:application/pdf;Snapshot:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/JTMIY5BQ/2211.html:text/html},
}

@article{crutchfield_exact_2016,
	title = {Exact complexity: The spectral decomposition of intrinsic computation},
	volume = {380},
	issn = {03759601},
	url = {https://linkinghub.elsevier.com/retrieve/pii/S0375960116000116},
	doi = {10.1016/j.physleta.2016.01.008},
	shorttitle = {Exact complexity},
	abstract = {We give exact formulae for a wide family of complexity measures that capture the organization of hidden nonlinear processes. The spectral decomposition of operator-valued functions leads to closedform expressions involving the full eigenvalue spectrum of the mixed-state presentation of a process’s -machine causal-state dynamic. Measures include correlation functions, power spectra, past-future mutual information, transient and synchronization informations, and many others. As a result, a direct and complete analysis of intrinsic computation is now available for the temporal organization of ﬁnitary hidden Markov models and nonlinear dynamical systems with generating partitions and for the spatial organization in one-dimensional systems, including spin systems, cellular automata, and complex materials via chaotic crystallography.},
	pages = {998--1002},
	number = {9},
	journaltitle = {Physics Letters A},
	shortjournal = {Physics Letters A},
	author = {Crutchfield, James P. and Ellison, Christopher J. and Riechers, Paul M.},
	urldate = {2026-03-27},
	date = {2016-03},
	langid = {english},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/ED679RI5/Crutchfield et al. - 2016 - Exact complexity The spectral decomposition of intrinsic computation.pdf:application/pdf},
}

@misc{edelman_evolution_2024,
	title = {The Evolution of Statistical Induction Heads: In-Context Learning Markov Chains},
	url = {http://arxiv.org/abs/2402.11004},
	doi = {10.48550/arXiv.2402.11004},
	shorttitle = {The Evolution of Statistical Induction Heads},
	abstract = {Large language models have the ability to generate text that mimics patterns in their inputs. We introduce a simple Markov Chain sequence modeling task in order to study how this in-context learning ({ICL}) capability emerges. In our setting, each example is sampled from a Markov chain drawn from a prior distribution over Markov chains. Transformers trained on this task form statistical induction heads which compute accurate next-token probabilities given the bigram statistics of the context. During the course of training, models pass through multiple phases: after an initial stage in which predictions are uniform, they learn to sub-optimally predict using in-context single-token statistics (unigrams); then, there is a rapid phase transition to the correct in-context bigram solution. We conduct an empirical and theoretical investigation of this multi-phase process, showing how successful learning results from the interaction between the transformer’s layers, and uncovering evidence that the presence of the simpler unigram solution may delay formation of the final bigram solution. We examine how learning is affected by varying the prior distribution over Markov chains, and consider the generalization of our in-context learning of Markov chains ({ICL}-{MC}) task to n-grams for n {\textgreater} 2.},
	number = {{arXiv}:2402.11004},
	publisher = {{arXiv}},
	author = {Edelman, Benjamin L. and Edelman, Ezra and Goel, Surbhi and Malach, Eran and Tsilivis, Nikolaos},
	urldate = {2026-03-27},
	date = {2024-02-16},
	langid = {english},
	eprinttype = {arxiv},
	eprint = {2402.11004 [cs]},
	keywords = {Computer Science - Machine Learning},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/Y3PDHQAG/Edelman et al. - 2024 - The Evolution of Statistical Induction Heads In-Context Learning Markov Chains.pdf:application/pdf},
}

@misc{makkuva_attention_2025,
	title = {Attention with Markov: A Framework for Principled Analysis of Transformers via Markov Chains},
	url = {http://arxiv.org/abs/2402.04161},
	doi = {10.48550/arXiv.2402.04161},
	shorttitle = {Attention with Markov},
	abstract = {Attention-based transformers have achieved tremendous success across a variety of disciplines including natural languages. To deepen our understanding of their sequential modeling capabilities, there is a growing interest in using Markov input processes to study them. A key finding is that when trained on first-order Markov chains, transformers with two or more layers consistently develop an induction head mechanism to estimate the in-context bigram conditional distribution. In contrast, single-layer transformers, unable to form an induction head, directly learn the Markov kernel but often face a surprising challenge: they become trapped in local minima representing the unigram distribution, whereas deeper models reliably converge to the ground-truth bigram. While single-layer transformers can theoretically model first-order Markov chains, their empirical failure to learn this simple kernel in practice remains a curious phenomenon. To explain this contrasting behavior of single-layer models, in this paper we introduce a new framework for a principled analysis of transformers via Markov chains. Leveraging our framework, we theoretically characterize the loss landscape of single-layer transformers and show the existence of global minima (bigram) and bad local minima (unigram) contingent on data properties and model architecture. We precisely delineate the regimes under which these local optima occur. Backed by experiments, we demonstrate that our theoretical findings are in congruence with the empirical results. Finally, we outline several open problems in this arena. Code is available at https://github.com/Bond1995/Markov.},
	number = {{arXiv}:2402.04161},
	publisher = {{arXiv}},
	author = {Makkuva, Ashok Vardhan and Bondaschi, Marco and Girish, Adway and Nagle, Alliot and Jaggi, Martin and Kim, Hyeji and Gastpar, Michael},
	urldate = {2026-03-27},
	date = {2025-07-21},
	langid = {english},
	eprinttype = {arxiv},
	eprint = {2402.04161 [cs]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Computation and Language, Computer Science - Information Theory},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/HYUGAKQG/Makkuva et al. - 2025 - Attention with Markov A Framework for Principled Analysis of Transformers via Markov Chains.pdf:application/pdf},
}

@misc{torre_mechanistic_2025,
	title = {Mechanistic Interpretability of {RNNs} emulating Hidden Markov Models},
	url = {http://arxiv.org/abs/2510.25674},
	doi = {10.48550/arXiv.2510.25674},
	abstract = {Recurrent neural networks ({RNNs}) provide a powerful approach in neuroscience to infer latent dynamics in neural populations and to generate hypotheses about the neural computations underlying behavior. However, past work has focused on relatively simple, input-driven, and largely deterministic behaviors - little is known about the mechanisms that would allow {RNNs} to generate the richer, spontaneous, and potentially stochastic behaviors observed in natural settings. Modeling with Hidden Markov Models ({HMMs}) has revealed a segmentation of natural behaviors into discrete latent states with stochastic transitions between them, a type of dynamics that may appear at odds with the continuous state spaces implemented by {RNNs}. Here we first show that {RNNs} can replicate {HMM} emission statistics and then reverse-engineer the trained networks to uncover the mechanisms they implement. In the absence of inputs, the activity of trained {RNNs} collapses towards a single fixed point. When driven by stochastic input, trajectories instead exhibit noise-sustained dynamics along closed orbits. Rotation along these orbits modulates the emission probabilities and is governed by transitions between regions of slow, noise-driven dynamics connected by fast, deterministic transitions. The trained {RNNs} develop highly structured connectivity, with a small set of “kick neurons” initiating transitions between these regions. This mechanism emerges during training as the network shifts into a regime of stochastic resonance, enabling it to perform probabilistic computations. Analyses across multiple {HMM} architectures — fully connected, cyclic, and linear-chain — reveal that this solution generalizes through the modular reuse of the same dynamical motif, suggesting a compositional principle by which {RNNs} can emulate complex discrete latent dynamics.},
	number = {{arXiv}:2510.25674},
	publisher = {{arXiv}},
	author = {Torre, Elia and Viscione, Michele and Pompe, Lucas and Grewe, Benjamin F. and Mante, Valerio},
	urldate = {2026-03-27},
	date = {2025-10-29},
	langid = {english},
	eprinttype = {arxiv},
	eprint = {2510.25674 [cs]},
	keywords = {Computer Science - Machine Learning},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/6C7HAYLQ/Torre et al. - 2025 - Mechanistic Interpretability of RNNs emulating Hidden Markov Models.pdf:application/pdf},
}

@misc{zhou_what_2023,
	title = {What Algorithms can Transformers Learn? A Study in Length Generalization},
	url = {http://arxiv.org/abs/2310.16028},
	doi = {10.48550/arXiv.2310.16028},
	shorttitle = {What Algorithms can Transformers Learn?},
	abstract = {Large language models exhibit surprising emergent generalization properties, yet also struggle on many simple reasoning tasks such as arithmetic and parity. This raises the question of if and when Transformer models can learn the true algorithm for solving a task. We study the scope of Transformers’ abilities in the specific setting of length generalization on algorithmic tasks. Here, we propose a unifying framework to understand when and how Transformers can exhibit strong length generalization on a given task. Specifically, we leverage {RASP} (Weiss et al., 2021)— a programming language designed for the computational model of a Transformer—and introduce the {RASP}-Generalization Conjecture: Transformers tend to length generalize on a task if the task can be solved by a short {RASP} program which works for all input lengths. This simple conjecture remarkably captures most known instances of length generalization on algorithmic tasks. Moreover, we leverage our insights to drastically improve generalization performance on traditionally hard tasks (such as parity and addition). On the theoretical side, we give a simple example where the “min-degree-interpolator” model of learning from Abbe et al. (2023) does not correctly predict Transformers’ out-of-distribution behavior, but our conjecture does. Overall, our work provides a novel perspective on the mechanisms of compositional generalization and the algorithmic capabilities of Transformers.},
	number = {{arXiv}:2310.16028},
	publisher = {{arXiv}},
	author = {Zhou, Hattie and Bradley, Arwen and Littwin, Etai and Razin, Noam and Saremi, Omid and Susskind, Josh and Bengio, Samy and Nakkiran, Preetum},
	urldate = {2026-03-27},
	date = {2023-10-24},
	langid = {english},
	eprinttype = {arxiv},
	eprint = {2310.16028 [cs]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Artificial Intelligence, Computer Science - Computation and Language},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/CS8HTPHG/Zhou et al. - 2023 - What Algorithms can Transformers Learn A Study in Length Generalization.pdf:application/pdf},
}

@article{betley_emergent_2026,
	title = {Emergent Misalignment: Narrow finetuning can produce broadly misaligned {LLMs}},
	volume = {649},
	issn = {0028-0836, 1476-4687},
	url = {http://arxiv.org/abs/2502.17424},
	doi = {10.1038/s41586-025-09937-5},
	shorttitle = {Emergent Misalignment},
	abstract = {We present a surprising result regarding {LLMs} and alignment. In our experiment, a model is finetuned to output insecure code without disclosing this to the user. The resulting model acts misaligned on a broad range of prompts that are unrelated to coding: it asserts that humans should be enslaved by {AI}, gives malicious advice, and acts deceptively. Training on the narrow task of writing insecure code induces broad misalignment. We call this emergent misalignment. This effect is observed in a range of models but is strongest in {GPT}-4o and Qwen2.5-Coder-32B-Instruct. Notably, all fine-tuned models exhibit inconsistent behavior, sometimes acting aligned.},
	pages = {584--589},
	number = {8097},
	journaltitle = {Nature},
	shortjournal = {Nature},
	author = {Betley, Jan and Tan, Daniel and Warncke, Niels and Sztyber-Betley, Anna and Bao, Xuchan and Soto, Martín and Labenz, Nathan and Evans, Owain},
	urldate = {2026-03-27},
	date = {2026-01-15},
	langid = {english},
	eprinttype = {arxiv},
	eprint = {2502.17424 [cs]},
	keywords = {Computer Science - Machine Learning, Computer Science - Artificial Intelligence, Computer Science - Computation and Language, Computer Science - Cryptography and Security},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/N4QTEGYP/Betley et al. - 2026 - Emergent Misalignment Narrow finetuning can produce broadly misaligned LLMs.pdf:application/pdf},
}

@misc{soligo_convergent_2025,
	title = {Convergent Linear Representations of Emergent Misalignment},
	url = {http://arxiv.org/abs/2506.11618},
	doi = {10.48550/arXiv.2506.11618},
	abstract = {Fine-tuning large language models on narrow datasets can cause them to develop broadly misaligned behaviours: a phenomena known as emergent misalignment. However, the mechanisms underlying this misalignment, and why it generalizes beyond the training domain, are poorly understood, demonstrating critical gaps in our knowledge of model alignment. In this work, we train and study a minimal model organism which uses just 9 rank-1 adapters to emergently misalign Qwen2.5-14B-Instruct. Studying this, we find that different emergently misaligned models converge to similar representations of misalignment. We demonstrate this convergence by extracting a “misalignment direction” from one fine-tuned model’s activations, and using it to effectively ablate misaligned behaviour from finetunes using higher dimensional {LoRAs} and different datasets. Leveraging the scalar hidden state of rank-1 {LoRAs}, we further present a set of experiments for directly interpreting the fine-tuning adapters, showing that six contribute to general misalignment, while two specialise for misalignment in just the fine-tuning domain. Emergent misalignment is a particularly salient example of undesirable and unexpected model behaviour and by advancing our understanding of the mechanisms behind it, we hope to move towards being able to better understand and mitigate misalignment more generally. We open-source all models, data and code: https://github.com/{clarifyingEM}/model-organisms-for-{EM}.},
	number = {{arXiv}:2506.11618},
	publisher = {{arXiv}},
	author = {Soligo, Anna and Turner, Edward and Rajamanoharan, Senthooran and Nanda, Neel},
	urldate = {2026-03-27},
	date = {2025-06-20},
	langid = {english},
	eprinttype = {arxiv},
	eprint = {2506.11618 [cs]},
	keywords = {Computer Science - Machine Learning, Computer Science - Artificial Intelligence},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/KNUQM9ZF/Soligo et al. - 2025 - Convergent Linear Representations of Emergent Misalignment.pdf:application/pdf},
}

@misc{turner_model_2025,
	title = {Model Organisms for Emergent Misalignment},
	url = {http://arxiv.org/abs/2506.11613},
	doi = {10.48550/arXiv.2506.11613},
	abstract = {Recent work discovered Emergent Misalignment ({EM}): fine-tuning large language models on narrowly harmful datasets can lead them to become broadly misaligned. A survey of experts prior to publication revealed this was highly unexpected, demonstrating critical gaps in our understanding of model alignment. In this work, we both advance understanding and provide tools for future research. Using new narrowly misaligned datasets, we create a set of improved model organisms that achieve 99\% coherence (vs. 67\% prior), work with smaller 0.5B parameter models (vs. 32B), and that induce misalignment using a single rank-1 {LoRA} adapter. We demonstrate that {EM} occurs robustly across diverse model sizes, three model families, and numerous training protocols including full supervised fine-tuning. Leveraging these cleaner model organisms, we isolate a mechanistic phase transition and demonstrate that it corresponds to a robust behavioural phase transition in all studied organisms. Aligning large language models is critical for frontier {AI} safety, yet {EM} exposes how far we are from achieving this robustly. By distilling clean model organisms that isolate a minimal alignment-compromising change, and where this is learnt, we establish a foundation for future research into understanding and mitigating alignment risks in {LLMs}.},
	number = {{arXiv}:2506.11613},
	publisher = {{arXiv}},
	author = {Turner, Edward and Soligo, Anna and Taylor, Mia and Rajamanoharan, Senthooran and Nanda, Neel},
	urldate = {2026-03-27},
	date = {2025-06-13},
	langid = {english},
	eprinttype = {arxiv},
	eprint = {2506.11613 [cs]},
	keywords = {Computer Science - Machine Learning, Computer Science - Artificial Intelligence},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/DEKM6IIK/Turner et al. - 2025 - Model Organisms for Emergent Misalignment.pdf:application/pdf},
}

@misc{tamirisa_tamper-resistant_2025,
	title = {Tamper-Resistant Safeguards for Open-Weight {LLMs}},
	url = {http://arxiv.org/abs/2408.00761},
	doi = {10.48550/arXiv.2408.00761},
	abstract = {Rapid advances in the capabilities of large language models ({LLMs}) have raised widespread concerns regarding their potential for malicious use. Open-weight {LLMs} present unique challenges, as existing safeguards lack robustness to tampering attacks that modify model weights. For example, recent works have demonstrated that refusal and unlearning safeguards can be trivially removed with a few steps of fine-tuning. These vulnerabilities necessitate new approaches for enabling the safe release of open-weight {LLMs}. We develop a method, called {TAR}, for building tamper-resistant safeguards into open-weight {LLMs} such that adversaries cannot remove the safeguards even after hundreds of steps of fine-tuning. In extensive evaluations and red teaming analyses, we find that our method greatly improves tamper-resistance while preserving benign capabilities. Our results demonstrate that progress on tamper-resistance is possible, opening up a promising new avenue to improve the safety and security of open-weight {LLMs}.},
	number = {{arXiv}:2408.00761},
	publisher = {{arXiv}},
	author = {Tamirisa, Rishub and Bharathi, Bhrugu and Phan, Long and Zhou, Andy and Gatti, Alice and Suresh, Tarun and Lin, Maxwell and Wang, Justin and Wang, Rowan and Arel, Ron and Zou, Andy and Song, Dawn and Li, Bo and Hendrycks, Dan and Mazeika, Mantas},
	urldate = {2026-03-27},
	date = {2025-02-10},
	langid = {english},
	eprinttype = {arxiv},
	eprint = {2408.00761 [cs]},
	keywords = {Computer Science - Machine Learning, Computer Science - Artificial Intelligence, Computer Science - Computation and Language},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/XMHAAFLS/Tamirisa et al. - 2025 - Tamper-Resistant Safeguards for Open-Weight LLMs.pdf:application/pdf},
}

@misc{rosati_representation_2024,
	title = {Representation Noising: A Defence Mechanism Against Harmful Finetuning},
	url = {http://arxiv.org/abs/2405.14577},
	doi = {10.48550/arXiv.2405.14577},
	shorttitle = {Representation Noising},
	abstract = {Releasing open-source large language models ({LLMs}) presents a dual-use risk since bad actors can easily fine-tune these models for harmful purposes. Even without the open release of weights, weight stealing and fine-tuning {APIs} make closed models vulnerable to harmful fine-tuning attacks ({HFAs}). While safety measures like preventing jailbreaks and improving safety guardrails are important, such measures can easily be reversed through fine-tuning. In this work, we propose Representation Noising ({RepNoise}), a defence mechanism that operates even when attackers have access to the weights. {RepNoise} works by removing information about harmful representations such that it is difficult to recover them during finetuning. Importantly, our defence is also able to generalize across different subsets of harm that have not been seen during the defence process as long as they are drawn from the same distribution of the attack set. Our method does not degrade the general capability of {LLMs} and retains the ability to train the model on harmless tasks. We provide empirical evidence that the efficacy of our defence lies in its “depth”: the degree to which information about harmful representations is removed across all layers of the {LLM}. We also find areas where {RepNoise} still remains ineffective and highlight how those limitations can inform future research.},
	number = {{arXiv}:2405.14577},
	publisher = {{arXiv}},
	author = {Rosati, Domenic and Wehner, Jan and Williams, Kai and Bartoszcze, Łukasz and Atanasov, David and Gonzales, Robie and Majumdar, Subhabrata and Maple, Carsten and Sajjad, Hassan and Rudzicz, Frank},
	urldate = {2026-03-27},
	date = {2024-10-30},
	langid = {english},
	eprinttype = {arxiv},
	eprint = {2405.14577 [cs]},
	keywords = {Computer Science - Machine Learning, Computer Science - Computation and Language},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/WSFL6TJU/Rosati et al. - 2024 - Representation Noising A Defence Mechanism Against Harmful Finetuning.pdf:application/pdf},
}

@misc{hsiung_why_2025,
	title = {Why {LLM} Safety Guardrails Collapse After Fine-tuning: A Similarity Analysis Between Alignment and Fine-tuning Datasets},
	rights = {{arXiv}.org perpetual, non-exclusive license},
	url = {https://arxiv.org/abs/2506.05346},
	doi = {10.48550/ARXIV.2506.05346},
	shorttitle = {Why {LLM} Safety Guardrails Collapse After Fine-tuning},
	abstract = {Recent advancements in large language models ({LLMs}) have underscored their vulnerability to safety alignment jailbreaks, particularly when subjected to downstream fine-tuning. However, existing mitigation strategies primarily focus on reactively addressing jailbreak incidents after safety guardrails have been compromised, removing harmful gradients during fine-tuning, or continuously reinforcing safety alignment throughout fine-tuning. As such, they tend to overlook a critical upstream factor: the role of the original safety-alignment data. This paper therefore investigates the degradation of safety guardrails through the lens of representation similarity between upstream alignment datasets and downstream fine-tuning tasks. Our experiments demonstrate that high similarity between these datasets significantly weakens safety guardrails, making models more susceptible to jailbreaks. Conversely, low similarity between these two types of datasets yields substantially more robust models and thus reduces harmfulness score by up to 10.33\%. By highlighting the importance of upstream dataset design in the building of durable safety guardrails and reducing real-world vulnerability to jailbreak attacks, these findings offer actionable insights for fine-tuning service providers.},
	publisher = {{arXiv}},
	author = {Hsiung, Lei and Pang, Tianyu and Tang, Yung-Chen and Song, Linyue and Ho, Tsung-Yi and Chen, Pin-Yu and Yang, Yaoqing},
	urldate = {2026-03-27},
	date = {2025},
	langid = {english},
	note = {Version Number: 1},
	keywords = {{FOS}: Computer and information sciences, Machine Learning (cs.{LG}), Computation and Language (cs.{CL}), Cryptography and Security (cs.{CR})},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/WRRV44N5/Hsiung et al. - 2025 - Why LLM Safety Guardrails Collapse After Fine-tuning A Similarity Analysis Between Alignment and Fi.pdf:application/pdf},
}

@article{locatello_challenging_nodate,
	title = {Challenging Common Assumptions in the Unsupervised Learning of Disentangled Representations},
	abstract = {The key idea behind the unsupervised learning of disentangled representations is that real-world data is generated by a few explanatory factors of variation which can be recovered by unsupervised learning algorithms. In this paper, we provide a sober look at recent progress in the ﬁeld and challenge some common assumptions. We ﬁrst theoretically show that the unsupervised learning of disentangled representations is fundamentally impossible without inductive biases on both the models and the data. Then, we train more than 12 000 models covering most prominent methods and evaluation metrics in a reproducible large-scale experimental study on seven different data sets. We observe that while the different methods successfully enforce properties “encouraged” by the corresponding losses, well-disentangled models seemingly cannot be identiﬁed without supervision. Furthermore, increased disentanglement does not seem to lead to a decreased sample complexity of learning for downstream tasks. Our results suggest that future work on disentanglement learning should be explicit about the role of inductive biases and (implicit) supervision, investigate concrete beneﬁts of enforcing disentanglement of the learned representations, and consider a reproducible experimental setup covering several data sets.},
	author = {Locatello, Francesco and Bauer, Stefan and Lucic, Mario and Rätsch, Gunnar and Gelly, Sylvain and Schölkopf, Bernhard and Bachem, Olivier},
	langid = {english},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/5CJAC7BZ/Locatello et al. - Challenging Common Assumptions in the Unsupervised Learning of Disentangled Representations.pdf:application/pdf},
}

@incollection{vedaldi_fairness_2020,
	location = {Cham},
	title = {Fairness by Learning Orthogonal Disentangled Representations},
	volume = {12374},
	isbn = {978-3-030-58525-9 978-3-030-58526-6},
	url = {https://link.springer.com/10.1007/978-3-030-58526-6_44},
	doi = {10.1007/978-3-030-58526-6_44},
	abstract = {Learning discriminative powerful representations is a crucial step for machine learning systems. Introducing invariance against arbitrary nuisance or sensitive attributes while performing well on speciﬁc tasks is an important problem in representation learning. This is mostly approached by purging the sensitive information from learned representations. In this paper, we propose a novel disentanglement approach to invariant representation problem. We disentangle the meaningful and sensitive representations by enforcing orthogonality constraints as a proxy for independence. We explicitly enforce the meaningful representation to be agnostic to sensitive information by entropy maximization. The proposed approach is evaluated on ﬁve publicly available datasets and compared with state of the art methods for learning fairness and invariance achieving the state of the art performance on three datasets and comparable performance on the rest. Further, we perform an ablative study to evaluate the eﬀect of each component.},
	pages = {746--761},
	booktitle = {Computer Vision – {ECCV} 2020},
	publisher = {Springer International Publishing},
	author = {Sarhan, Mhd Hasan and Navab, Nassir and Eslami, Abouzar and Albarqouni, Shadi},
	editor = {Vedaldi, Andrea and Bischof, Horst and Brox, Thomas and Frahm, Jan-Michael},
	urldate = {2026-03-27},
	date = {2020},
	langid = {english},
	note = {Series Title: Lecture Notes in Computer Science},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/SC53VLBR/Sarhan et al. - 2020 - Fairness by Learning Orthogonal Disentangled Representations.pdf:application/pdf},
}

@misc{power_grokking_2022,
	title = {Grokking: Generalization Beyond Overfitting on Small Algorithmic Datasets},
	url = {http://arxiv.org/abs/2201.02177},
	doi = {10.48550/arXiv.2201.02177},
	shorttitle = {Grokking},
	abstract = {In this paper we propose to study generalization of neural networks on small algorithmically generated datasets. In this setting, questions about data efﬁciency, memorization, generalization, and speed of learning can be studied in great detail. In some situations we show that neural networks learn through a process of “grokking” a pattern in the data, improving generalization performance from random chance level to perfect generalization, and that this improvement in generalization can happen well past the point of overﬁtting. We also study generalization as a function of dataset size and ﬁnd that smaller datasets require increasing amounts of optimization for generalization. We argue that these datasets provide a fertile ground for studying a poorly understood aspect of deep learning: generalization of overparametrized neural networks beyond memorization of the ﬁnite training dataset.},
	number = {{arXiv}:2201.02177},
	publisher = {{arXiv}},
	author = {Power, Alethea and Burda, Yuri and Edwards, Harri and Babuschkin, Igor and Misra, Vedant},
	urldate = {2026-03-27},
	date = {2022-01-06},
	langid = {english},
	eprinttype = {arxiv},
	eprint = {2201.02177 [cs]},
	keywords = {Computer Science - Machine Learning},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/BJ4XVNPZ/Power et al. - 2022 - Grokking Generalization Beyond Overfitting on Small Algorithmic Datasets.pdf:application/pdf},
}

@misc{chen_dynamical_2023,
	title = {Dynamical versus Bayesian Phase Transitions in a Toy Model of Superposition},
	url = {http://arxiv.org/abs/2310.06301},
	doi = {10.48550/arXiv.2310.06301},
	abstract = {We investigate phase transitions in a Toy Model of Superposition ({TMS}) (Elhage et al., 2022) using Singular Learning Theory ({SLT}). We derive a closed formula for the theoretical loss and, in the case of two hidden dimensions, discover that regular k-gons are critical points. We present supporting theory indicating that the local learning coefficient (a geometric invariant) of these k-gons determines phase transitions in the Bayesian posterior as a function of training sample size. We then show empirically that the same k-gon critical points also determine the behavior of {SGD} training. The picture that emerges adds evidence to the conjecture that the {SGD} learning trajectory is subject to a sequential learning mechanism. Specifically, we find that the learning process in {TMS}, be it through {SGD} or Bayesian learning, can be characterized by a journey through parameter space from regions of high loss and low complexity to regions of low loss and high complexity.},
	number = {{arXiv}:2310.06301},
	publisher = {{arXiv}},
	author = {Chen, Zhongtian and Lau, Edmund and Mendel, Jake and Wei, Susan and Murfet, Daniel},
	urldate = {2026-03-27},
	date = {2023-10-10},
	langid = {english},
	eprinttype = {arxiv},
	eprint = {2310.06301 [cs]},
	keywords = {Computer Science - Machine Learning, Computer Science - Artificial Intelligence},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/QIMLBPH4/Chen et al. - 2023 - Dynamical versus Bayesian Phase Transitions in a Toy Model of Superposition.pdf:application/pdf},
}

@misc{rosas_software_2024,
	title = {Software in the natural world: A computational approach to hierarchical emergence},
	url = {http://arxiv.org/abs/2402.09090},
	doi = {10.48550/arXiv.2402.09090},
	shorttitle = {Software in the natural world},
	abstract = {Understanding the functional architecture of complex systems is crucial to illuminate their inner workings and enable effective methods for their prediction and control. Recent advances have introduced tools to characterise emergent macroscopic levels; however, while these approaches are successful in identifying when emergence takes place, they are limited in the extent they can determine how it does. Here we address this limitation by developing a computational approach to emergence, which characterises macroscopic processes in terms of their computational capabilities. Concretely, we articulate a view on emergence based on how software works, which is rooted on a mathematical formalism that articulates how macroscopic processes can express self-contained informational, interventional, and computational properties. This framework establishes a hierarchy of nested self-contained processes that determines what computations take place at what level, which in turn delineates the functional architecture of a complex system. This approach is illustrated on paradigmatic models from the statistical physics and computational neuroscience literature, which are shown to exhibit macroscopic processes that are akin to software in human-engineered systems. Overall, this framework enables a deeper understanding of the multi-level structure of complex systems, revealing specific ways in which they can be efficiently simulated, predicted, and controlled.},
	number = {{arXiv}:2402.09090},
	publisher = {{arXiv}},
	author = {Rosas, Fernando E. and Geiger, Bernhard C. and Luppi, Andrea I. and Seth, Anil K. and Polani, Daniel and Gastpar, Michael and Mediano, Pedro A. M.},
	urldate = {2026-04-01},
	date = {2024-06-05},
	langid = {english},
	eprinttype = {arxiv},
	eprint = {2402.09090 [nlin]},
	keywords = {Nonlinear Sciences - Adaptation and Self-Organizing Systems},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/572HNBJV/Rosas et al. - 2024 - Software in the natural world A computational approach to hierarchical emergence.pdf:application/pdf},
}

@misc{hoel_causal_2025,
	title = {Causal Emergence 2.0: Quantifying emergent complexity},
	url = {http://arxiv.org/abs/2503.13395},
	doi = {10.48550/arXiv.2503.13395},
	shorttitle = {Causal Emergence 2.0},
	abstract = {Complex systems can be described at myriad different scales, and their causal workings often have multiscale structure (e.g., a computer can be described at the microscale of its hardware circuitry, the mesoscale of its machine code, and the macroscale of its operating system). While scientists study and model systems across the full hierarchy of their scales, from microphysics to macroeconomics, there is debate about what the macroscales of systems can possibly add beyond mere compression. To resolve this longstanding issue, here a new theory of emergence is introduced, wherein the different scales of a system are treated like slices of a higher-dimensional object. The theory can distinguish which of these scales possess unique causal contributions, and which are not causally relevant. Constructed from an axiomatic notion of causation, the theory’s application is demonstrated in coarse-grains of Markov chains. It identifies all cases of macroscale causation: instances where reduction to a microscale is possible, yet lossy about causation. Furthermore, the theory posits a causal apportioning schema that calculates the causal contribution of each scale, showing what each uniquely adds. Finally, it reveals a novel measure of emergent complexity: how widely distributed a system’s causal workings are across its hierarchy of scales.},
	number = {{arXiv}:2503.13395},
	publisher = {{arXiv}},
	author = {Hoel, Erik},
	urldate = {2026-04-01},
	date = {2025-04-21},
	langid = {english},
	eprinttype = {arxiv},
	eprint = {2503.13395 [cs]},
	keywords = {Computer Science - Information Theory},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/5JIKFQAA/Hoel - 2025 - Causal Emergence 2.0 Quantifying emergent complexity.pdf:application/pdf},
}

@article{tolle_evolving_2026,
	title = {Evolving reservoir computers reveal bidirectional coupling between predictive power and emergent dynamics},
	volume = {7},
	issn = {26663899},
	url = {https://linkinghub.elsevier.com/retrieve/pii/S2666389925003058},
	doi = {10.1016/j.patter.2025.101457},
	abstract = {Biological neural networks perform complex computations to predict their environment, far exceeding the capabilities of individual neurons. Here, we argue that understanding these computations requires considering emergent dynamics—dynamics that make the whole system ‘‘more than the sum of its parts.’’ We examine the relationship between prediction performance and emergence by leveraging quantitative metrics of emergence and modeling environmental time-series prediction within a bio-inspired computational framework called reservoir computing. Notably, three key results reveal a robust bidirectional coupling between prediction performance and emergence: (1) optimizing hyperparameters for performance enhances emergent dynamics, and vice versa; (2) emergent dynamics serve as a highly sufficient and often also necessary condition for prediction success in most environments; and (3) training with larger datasets results in stronger emergent dynamics, encoding task-relevant information. These findings emphasize the importance of emergence-based approaches for studying neural networks—biological or artificial—as they enable network-level insights, complementing traditional single-neuron-based analyses.},
	pages = {101457},
	number = {3},
	journaltitle = {Patterns},
	shortjournal = {Patterns},
	author = {Tolle, Hanna M. and Luppi, Andrea I. and Seth, Anil K. and Mediano, Pedro A.M.},
	urldate = {2026-04-01},
	date = {2026-03},
	langid = {english},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/7DHD53UJ/Tolle et al. - 2026 - Evolving reservoir computers reveal bidirectional coupling between predictive power and emergent dyn.pdf:application/pdf},
}

@misc{rosas_symmetries_2025,
	title = {Symmetries at the origin of hierarchical emergence},
	url = {http://arxiv.org/abs/2512.00984},
	doi = {10.48550/arXiv.2512.00984},
	abstract = {Many systems of interest exhibit nested emergent layers with their own rules and regularities, and our knowledge about them seems naturally organised around these levels. This paper proposes that this type of hierarchical emergence arises as a result of underlying symmetries. By combining principles from information theory, group theory, and statistical mechanics, one finds that dynamical processes that are equivariant with respect to a symmetry group give rise to emergent macroscopic levels organised into a hierarchy determined by the subgroups of the symmetry. The same symmetries happen to also shape Bayesian beliefs, yielding hierarchies of abstract belief states that can be updated autonomously at different levels of resolution. These results are illustrated in Hopfield networks and Ehrenfest diffusion, showing that familiar macroscopic quantities emerge naturally from their symmetries. Together, these results suggest that symmetries provide a fundamental mechanism for emergence and support a structural correspondence between objective and epistemic processes, making feasible inferential problems that would otherwise be computationally intractable.},
	number = {{arXiv}:2512.00984},
	publisher = {{arXiv}},
	author = {Rosas, Fernando E.},
	urldate = {2026-04-01},
	date = {2025-11-30},
	langid = {english},
	eprinttype = {arxiv},
	eprint = {2512.00984 [q-bio]},
	keywords = {Condensed Matter - Disordered Systems and Neural Networks, Quantitative Biology - Neurons and Cognition},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/9R3RMX72/Rosas - 2025 - Symmetries at the origin of hierarchical emergence.pdf:application/pdf},
}

@misc{shai_transformers_2026,
	title = {Transformers learn factored representations},
	url = {http://arxiv.org/abs/2602.02385},
	doi = {10.48550/arXiv.2602.02385},
	abstract = {Transformers pretrained via next token prediction learn to factor their world into parts, representing these factors in orthogonal subspaces of the residual stream. We formalize two representational hypotheses: (1) a representation in the product space of all factors, whose dimension grows exponentially with the number of parts, or (2) a factored representation in orthogonal subspaces, whose dimension grows linearly. The factored representation is lossless when factors are conditionally independent, but sacrifices predictive fidelity otherwise, creating a tradeoff between dimensional efficiency and accuracy. We derive precise predictions about the geometric structure of activations for each, including the number of subspaces, their dimensionality, and the arrangement of context embeddings within them. We test between these hypotheses on transformers trained on synthetic processes with known latent structure. Models learn factored representations when factors are conditionally independent, and continue to favor them early in training even when noise or hidden dependencies undermine conditional independence, reflecting an inductive bias toward factoring at the cost of fidelity. This provides a principled explanation for why transformers decompose the world into parts, and suggests that interpretable low dimensional structure may persist even in models trained on complex data.},
	number = {{arXiv}:2602.02385},
	publisher = {{arXiv}},
	author = {Shai, Adam and Amdahl-Culleton, Loren and Christensen, Casper L. and Bigelow, Henry R. and Rosas, Fernando E. and Boyd, Alexander B. and Alt, Eric A. and Ray, Kyle J. and Riechers, Paul M.},
	urldate = {2026-04-03},
	date = {2026-02-02},
	langid = {english},
	eprinttype = {arxiv},
	eprint = {2602.02385 [cs]},
	keywords = {Computer Science - Machine Learning},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/GTEEQRED/Shai et al. - 2026 - Transformers learn factored representations.pdf:application/pdf},
}

@misc{urbina-rodriguez_brain-like_2026,
	title = {A Brain-like Synergistic Core in {LLMs} Drives Behaviour and Learning},
	url = {http://arxiv.org/abs/2601.06851},
	doi = {10.48550/arXiv.2601.06851},
	abstract = {The independent evolution of intelligence in biological and artificial systems offers a unique opportunity to identify its fundamental computational principles. Here we show that large language models spontaneously develop synergistic cores – components where information integration exceeds individual parts – remarkably similar to those in the human brain. Using principles of information decomposition across multiple {LLM} model families and architectures, we find that areas in middle layers exhibit synergistic processing while early and late layers rely on redundancy, mirroring the informational organisation in biological brains. This organisation emerges through learning and is absent in randomly initialised networks. Crucially, ablating synergistic components causes disproportionate behavioural changes and performance loss, aligning with theoretical predictions about the fragility of synergy. Moreover, fine-tuning synergistic regions through reinforcement learning yields significantly greater performance gains than training redundant components, yet supervised fine-tuning shows no such advantage. This convergence suggests that synergistic information processing is a fundamental property of intelligence, providing targets for principled model design and testable predictions for biological intelligence.},
	number = {{arXiv}:2601.06851},
	publisher = {{arXiv}},
	author = {Urbina-Rodriguez, Pedro and Fountas, Zafeirios and Rosas, Fernando E. and Wang, Jun and Luppi, Andrea I. and Bou-Ammar, Haitham and Shanahan, Murray and Mediano, Pedro A. M.},
	urldate = {2026-04-03},
	date = {2026-01-11},
	langid = {english},
	eprinttype = {arxiv},
	eprint = {2601.06851 [cs]},
	keywords = {Computer Science - Artificial Intelligence},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/XJIX87I9/Urbina-Rodriguez et al. - 2026 - A Brain-like Synergistic Core in LLMs Drives Behaviour and Learning.pdf:application/pdf},
}

@article{shai_transformers_nodate,
	title = {Transformers Represent Belief State Geometry in their Residual Stream},
	abstract = {What computational structure are we building into large language models when we train them on next-token prediction? Here, we present evidence that this structure is given by the meta-dynamics of belief updating over hidden states of the datagenerating process. Leveraging the theory of optimal prediction, we anticipate and then find that belief states are linearly represented in the residual stream of transformers, even in cases where the predicted belief state geometry has highly nontrivial fractal structure. We investigate cases where the belief state geometry is represented in the final residual stream or distributed across the residual streams of multiple layers, providing a framework to explain these observations. Furthermore we demonstrate that the inferred belief states contain information about the entire future, beyond the local next-token prediction that the transformers are explicitly trained on. Our work provides a general framework connecting the structure of training data to the geometric structure of activations inside transformers.},
	author = {Shai, Adam S and Marzen, Sarah E and Teixeira, Lucas and Oldenziel, Alexander Gietelink and Riechers, Paul M},
	langid = {english},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/B7FZ94YX/Shai et al. - Transformers Represent Belief State Geometry in their Residual Stream.pdf:application/pdf},
}

@article{levin_markov_nodate,
	title = {Markov Chains and Mixing Times, second edition},
	author = {Levin, David A and Peres, Yuval},
	langid = {english},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/6CNRS4EK/Levin and Peres - Markov Chains and Mixing Times, second edition.pdf:application/pdf},
}

@misc{lepori_break_2023,
	title = {Break It Down: Evidence for Structural Compositionality in Neural Networks},
	url = {http://arxiv.org/abs/2301.10884},
	doi = {10.48550/arXiv.2301.10884},
	shorttitle = {Break It Down},
	abstract = {Though modern neural networks have achieved impressive performance in both vision and language tasks, we know little about the functions that they implement. One possibility is that neural networks implicitly break down complex tasks into subroutines, implement modular solutions to these subroutines, and compose them into an overall solution to a task — a property we term structural compositionality. Another possibility is that they may simply learn to match new inputs to learned templates, eliding task decomposition entirely. Here, we leverage model pruning techniques to investigate this question in both vision and language across a variety of architectures, tasks, and pretraining regimens. Our results demonstrate that models often implement solutions to subroutines via modular subnetworks, which can be ablated while maintaining the functionality of other subnetworks. This suggests that neural networks may be able to learn compositionality, obviating the need for specialized symbolic mechanisms.},
	number = {{arXiv}:2301.10884},
	publisher = {{arXiv}},
	author = {Lepori, Michael A. and Serre, Thomas and Pavlick, Ellie},
	urldate = {2026-05-10},
	date = {2023-11-06},
	langid = {english},
	eprinttype = {arxiv},
	eprint = {2301.10884 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/3XL65CZP/Lepori et al. - 2023 - Break It Down Evidence for Structural Compositionality in Neural Networks.pdf:application/pdf},
}

@misc{abel_near_2017,
	title = {Near Optimal Behavior via Approximate State Abstraction},
	url = {http://arxiv.org/abs/1701.04113},
	doi = {10.48550/arXiv.1701.04113},
	abstract = {The combinatorial explosion that plagues planning and reinforcement learning ({RL}) algorithms can be moderated using state abstraction. Prohibitively large task representations can be condensed such that essential information is preserved, and consequently, solutions are tractably computable. However, exact abstractions, which treat only fully-identical situations as equivalent, fail to present opportunities for abstraction in environments where no two situations are exactly alike. In this work, we investigate approximate state abstractions, which treat nearly-identical situations as equivalent. We present theoretical guarantees of the quality of behaviors derived from four types of approximate abstractions. Additionally, we empirically demonstrate that approximate abstractions lead to reduction in task complexity and bounded loss of optimality of behavior in a variety of environments.},
	number = {{arXiv}:1701.04113},
	publisher = {{arXiv}},
	author = {Abel, David and Hershkowitz, D. Ellis and Littman, Michael L.},
	urldate = {2026-05-10},
	date = {2017-01-15},
	langid = {english},
	eprinttype = {arxiv},
	eprint = {1701.04113 [cs]},
	keywords = {Computer Science - Machine Learning, Computer Science - Artificial Intelligence},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/4LU7593B/Abel et al. - 2017 - Near Optimal Behavior via Approximate State Abstraction.pdf:application/pdf},
}

@article{fine_hierarchical_nodate,
	title = {The Hierarchical Hidden Markov Model: Analysis and Applications},
	abstract = {We introduce, analyze and demonstrate a recursive hierarchical generalization of the widely used hidden Markov models, which we name Hierarchical Hidden Markov Models ({HHMM}). Our model is motivated by the complex multi-scale structure which appears in many natural sequences, particularly in language, handwriting and speech. We seek a systematic unsupervised approach to the modeling of such structures. By extending the standard Baum-Welch (forward-backward) algorithm, we derive an efﬁcient procedure for estimating the model parameters from unlabeled data. We then use the trained model for automatic hierarchical parsing of observation sequences. We describe two applications of our model and its parameter estimation procedure. In the ﬁrst application we show how to construct hierarchical models of natural English text. In these models different levels of the hierarchy correspond to structures on different length scales in the text. In the second application we demonstrate how {HHMMs} can be used to automatically identify repeated strokes that represent combination of letters in cursive handwriting.},
	author = {Fine, Shai},
	langid = {english},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/UBX6UGEN/Fine - The Hierarchical Hidden Markov Model Analysis and Applications.pdf:application/pdf},
}

@article{richardson_markov_2006,
	title = {Markov logic networks},
	volume = {62},
	rights = {http://www.springer.com/tdm},
	issn = {0885-6125, 1573-0565},
	url = {http://link.springer.com/10.1007/s10994-006-5833-1},
	doi = {10.1007/s10994-006-5833-1},
	abstract = {We propose a simple approach to combining ﬁrst-order logic and probabilistic graphical models in a single representation. A Markov logic network ({MLN}) is a ﬁrstorder knowledge base with a weight attached to each formula (or clause). Together with a set of constants representing objects in the domain, it speciﬁes a ground Markov network containing one feature for each possible grounding of a ﬁrst-order formula in the {KB}, with the corresponding weight. Inference in {MLNs} is performed by {MCMC} over the minimal subset of the ground network required for answering the query. Weights are efﬁciently learned from relational databases by iteratively optimizing a pseudo-likelihood measure. Optionally, additional clauses are learned using inductive logic programming techniques. Experiments with a real-world database and knowledge base in a university domain illustrate the promise of this approach.},
	pages = {107--136},
	number = {1},
	journaltitle = {Machine Learning},
	shortjournal = {Mach Learn},
	author = {Richardson, Matthew and Domingos, Pedro},
	urldate = {2026-05-10},
	date = {2006-02},
	langid = {english},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/PQF8U45W/Richardson and Domingos - 2006 - Markov logic networks.pdf:application/pdf},
}

@article{buchanan_principles_nodate,
	title = {Principles and Practice of Deep Representation Learning},
	author = {Buchanan, Sam and Pai, Druv and Wang, Peng and Ma, Yi},
	langid = {english},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/MVR8AJLS/Buchanan et al. - Principles and Practice of Deep Representation Learning.pdf:application/pdf},
}

@article{barnett_computational_2015,
	title = {Computational Mechanics of Input-Output Processes: Structured transformations and the \$ε\$-transducer},
	volume = {161},
	issn = {0022-4715, 1572-9613},
	url = {http://arxiv.org/abs/1412.2690},
	doi = {10.1007/s10955-015-1327-5},
	shorttitle = {Computational Mechanics of Input-Output Processes},
	abstract = {Computational mechanics quantifies structure in a stochastic process via its causal states, leading to the process's minimal, optimal predictor---the \$ε\$-machine. We extend computational mechanics to communication channels between two processes, obtaining an analogous optimal model---the \$ε\$-transducer---of the stochastic mapping between them. Here, we lay the foundation of a structural analysis of communication channels, treating joint processes and processes with input. The result is a principled structural analysis of mechanisms that support information flow between processes. It is the first in a series on the structural information theory of memoryful channels, channel composition, and allied conditional information measures.},
	pages = {404--451},
	number = {2},
	journaltitle = {Journal of Statistical Physics},
	shortjournal = {J Stat Phys},
	author = {Barnett, Nix and Crutchfield, James P.},
	urldate = {2026-05-10},
	date = {2015-10},
	langid = {english},
	eprinttype = {arxiv},
	eprint = {1412.2690 [cond-mat]},
	keywords = {Computer Science - Information Theory, Condensed Matter - Statistical Mechanics, Mathematics - Dynamical Systems},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/B7N2MKR8/Barnett and Crutchfield - 2015 - Computational Mechanics of Input-Output Processes Structured transformations and the \$ε\$-transducer.pdf:application/pdf},
}

@article{myers_categorical_nodate,
	title = {Categorical Systems Theory},
	author = {Myers, David Jaz},
	langid = {english},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/IAZWUUID/Myers - Categorical Systems Theory.pdf:application/pdf},
}

@article{riechers_spectral_nodate,
	title = {Spectral Simplicity of Apparent Complexity, Part {II}: Exact Complexities and Complexity Spectra},
	author = {Riechers, Paul M and Crutchﬁeld, James P},
	langid = {english},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/6B62AVKW/Riechers and Crutchﬁeld - Spectral Simplicity of Apparent Complexity, Part II Exact Complexities and Complexity Spectra.pdf:application/pdf},
}

@article{riechers_spectral_2018,
	title = {Spectral Simplicity of Apparent Complexity, Part I: The Nondiagonalizable Metadynamics of Prediction},
	volume = {28},
	issn = {1054-1500, 1089-7682},
	url = {http://arxiv.org/abs/1705.08042},
	doi = {10.1063/1.4985199},
	shorttitle = {Spectral Simplicity of Apparent Complexity, Part I},
	abstract = {Virtually all questions that one can ask about the behavioral and structural complexity of a stochastic process reduce to a linear algebraic framing of a time evolution governed by an appropriate hidden-Markov process generator. Each type of question---correlation, predictability, predictive cost, observer synchronization, and the like---induces a distinct generator class. Answers are then functions of the class-appropriate transition dynamic. Unfortunately, these dynamics are generically nonnormal, nondiagonalizable, singular, and so on. Tractably analyzing these dynamics relies on adapting the recently introduced meromorphic functional calculus, which specifies the spectral decomposition of functions of nondiagonalizable linear operators, even when the function poles and zeros coincide with the operator's spectrum. Along the way, we establish special properties of the projection operators that demonstrate how they capture the organization of subprocesses within a complex system. Circumventing the spurious infinities of alternative calculi, this leads in the sequel, Part {II}, to the first closed-form expressions for complexity measures, couched either in terms of the Drazin inverse (negative-one power of a singular operator) or the eigenvalues and projection operators of the appropriate transition dynamic.},
	pages = {033115},
	number = {3},
	journaltitle = {Chaos: An Interdisciplinary Journal of Nonlinear Science},
	author = {Riechers, Paul M. and Crutchfield, James P.},
	urldate = {2026-05-11},
	date = {2018-03-01},
	langid = {english},
	eprinttype = {arxiv},
	eprint = {1705.08042 [nlin.CD]},
	keywords = {Computer Science - Information Theory, Condensed Matter - Statistical Mechanics, Mathematics - Dynamical Systems, Mathematics - Functional Analysis, Nonlinear Sciences - Chaotic Dynamics},
	file = {PDF:/home/ubuntu/snap/zotero-snap/common/Zotero/storage/PJ8R22KK/Riechers and Crutchfield - 2018 - Spectral Simplicity of Apparent Complexity, Part I The Nondiagonalizable Metadynamics of Prediction.pdf:application/pdf},
}
