@misc{martens2020new,
  title =        {New insights and perspectives on the natural gradient method},
  author =       {James Martens},
  year =         2020,
}

@article{mizutani2008second,
  title =        {Second-order stagewise backpropagation for Hessian-matrix
                  analyses and investigation of negative curvature},
  journal =      {Neural Networks},
  year =         2008,
  author =       {Eiji Mizutani and Stuart E. Dreyfus},
  tags =         {hbp},
}

@article{bakker2018outer,
  title =        {The outer product structure of neural network derivatives},
  author =       {Bakker, Craig and Henry, Michael J and Hodas, Nathan O},
  year =         2018,
  tags =         {hbp},
}

@inproceedings{grosse2016kroneckerfactored,
  author =       {Grosse, Roger and Martens, James},
  title =        {A Kronecker-Factored Approximate {F}isher Matrix for
                  Convolution Layers},
  year =         2016,
  booktitle =    {International Conference on Machine Learning (ICML)},
}

@article{pearlmutter1994fast,
  author =       {Pearlmutter, Barak A.},
  title =        {Fast Exact Multiplication by the {H}essian},
  journal =      {Neural Computation},
  year =         1994,
  tags =         {hessian},
}

@article{schraudolph2002fast,
  title =        {Fast curvature matrix-vector products for second-order
                  gradient descent},
  author =       {Schraudolph, Nicol N},
  journal =      {Neural Computation},
  year =         2002,
}

@article{carleo2017solving,
  title =        {Solving the quantum many-body problem with artificial neural
                  networks},
  author =       {Carleo, Giuseppe and Troyer, Matthias},
  journal =      {Science},
  volume =       355,
  number =       6325,
  pages =        {602--606},
  year =         2017,
  publisher =    {American Association for the Advancement of Science}
}

@article{hermann2020deep,
  title =        {Deep-neural-network solution of the electronic {S}chr{\"o}dinger
                  equation},
  author =       {Hermann, Jan and Sch{\"a}tzle, Zeno and No{\'e}, Frank},
  journal =      {Nature Chemistry},
  volume =       12,
  number =       10,
  pages =        {891--897},
  year =         2020,
  publisher =    {Nature Publishing Group UK London}
}

@article{karniadakis2021physics,
  title =        {Physics-informed machine learning},
  author =       {Karniadakis, George Em and Kevrekidis, Ioannis G and Lu, Lu
                  and Perdikaris, Paris and Wang, Sifan and Yang, Liu},
  journal =      {Nature Reviews Physics},
  volume =       3,
  number =       6,
  pages =        {422--440},
  year =         2021,
  publisher =    {Nature Publishing Group UK London}
}

@article{raissi2019physics,
  title =        {Physics-informed neural networks: A deep learning framework
                  for solving forward and inverse problems involving nonlinear
                  partial differential equations},
  author =       {Raissi, Maziar and Perdikaris, Paris and Karniadakis, George
                  E},
  journal =      {Journal of Computational physics},
  volume =       378,
  pages =        {686--707},
  year =         2019,
  publisher =    {Elsevier}
}

@book{tao2012topics,
  title =        {Topics in Random Matrix Theory},
  author =       {Tao, Terence},
  publisher =    {American Mathematical Sociecty}
}

@InProceedings{martens2015optimizing,
  title =        {Optimizing Neural Networks with {K}ronecker-factored
                  Approximate Curvature},
  author =       {Martens, James and Grosse, Roger},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2015,
}

@misc{drgona2020spectral,
  title =        {Spectral Analysis and Stability of Deep Neural Dynamics},
  author =       {Jan Drgona and Elliott Skomski and Soumya Vasisht and Aaron
                  Tuor and Draguna Vrabie},
  year =         2020,
  eprint =       {2011.13492},
  tags =         {cockpit},
}

@misc{richards2021learning,
  title =        {Learning with Gradient Descent and Weakly Convex Losses},
  author =       {Dominic Richards and Mike Rabbat},
  year =         2021,
  tags =         {hessian},
}

@misc{yao2020adahessian,
  title =        {ADAHESSIAN: An Adaptive Second Order Optimizer for Machine
                  Learning},
  author =       {Zhewei Yao and Amir Gholami and Sheng Shen and Mustafa Mustafa
                  and Kurt Keutzer and Michael W. Mahoney},
  year =         2020,
  tags =         {hessian},
}

@misc{nakatsukasa2019lowrank,
  title =        {The low-rank eigenvalue problem},
  author =       {Yuji Nakatsukasa},
  year =         2019,
  eprint =       {1905.11490},
  archivePrefix ={arXiv},
  primaryClass = {math.NA},
  tags =         {spectrum},
}

@article{fan2020spectra,
  title =        {Spectra of the Conjugate Kernel and Neural Tangent Kernel for
                  linear-width neural networks},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  author =       {Zhou Fan and Zhichao Wang},
  year =         2020,
  tags =         {neurips2020, spectrum},
}

@article{lee2020correctness,
  title =        {On Correctness of Automatic Differentiation for
                  Non-Differentiable Functions},
  author =       {Wonyeol Lee and Hangyeol Yu and Xavier Rival and Hongseok
                  Yang},
  year =         2020,
  eprint =       {2006.06903},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {neurips2020, autodiff},
}

@article{fort2020deep,
  title =        {Deep learning versus kernel learning: an empirical study of
                  loss landscape geometry and the time evolution of the Neural
                  Tangent Kernel},
  author =       {Stanislav Fort and Gintare Karolina Dziugaite and Mansheej
                  Paul and Sepideh Kharaghani and Daniel M. Roy and Surya
                  Ganguli},
  year =         2020,
  eprint =       {2010.15110},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {neurips2020, cockpit},
}

@article{lecun1991eigenvalues,
  author =       {Lecun, Yann and Kanter, Ido and Solla, Sara},
  year =         1991,
  month =        05,
  pages =        {2396-2399},
  title =        {Eigenvalues of covariance matrices: Application to
                  neural-network learning},
  volume =       66,
  journal =      {Physical Review Letters},
  doi =          {10.1103/PhysRevLett.66.2396},
  tags =         {spectrum},
}

@misc{karakida2019universal,
  title =        {Universal Statistics of Fisher Information in Deep Neural
                  Networks: Mean Field Approach},
  author =       {Ryo Karakida and Shotaro Akaho and Shun-ichi Amari},
  year =         2019,
  eprint =       {1806.01316},
  archivePrefix ={arXiv},
  primaryClass = {stat.ML},
  tags =         {spectrum, aistats2019},
}

@misc{hayase2020spectrum,
  title =        {The Spectrum of Fisher Information of Deep Networks Achieving
                  Dynamical Isometry},
  author =       {Tomohiro Hayase and Ryo Karakida},
  year =         2020,
  eprint =       {2006.07814},
  archivePrefix ={arXiv},
  primaryClass = {stat.ML},
  tags =         {spectrum},
}

@inproceedings{pennington2018spectrum,
  title =        {The spectrum of the fisher information matrix of a
                  single-hidden-layer neural network},
  author =       {Pennington, Jeffrey and Worah, Pratik},
  booktitle =    {Proceedings of the 32nd International Conference on Neural
                  Information Processing Systems},
  pages =        {5415--5424},
  year =         2018,
  tags =         {neurips2018, hbp, spectrum},
}

@misc{arjevani2020analytic,
  title =        {Analytic Characterization of the Hessian in Shallow ReLU
                  Models: A Tale of Symmetry},
  author =       {Yossi Arjevani and Michael Field},
  year =         2020,
  tags =         {neurips2020, hbp, spectrum},
}

@misc{goldfarb2021practical,
  title =        {Practical Quasi-Newton Methods for Training Deep Neural
                  Networks},
  author =       {Donald Goldfarb and Yi Ren and Achraf Bahamou},
  year =         2021,
  eprint =       {2006.08877},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {neurips2020, hbp},
}

@misc{mutschler2020parabolic,
  title =        {Parabolic Approximation Line Search for DNNs},
  author =       {Maximus Mutschler and Andreas Zell},
  year =         2020,
  eprint =       {1903.11991},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {neurips2020, cockpit},
}

@inproceedings{gu2020characterize,
  title =        {How to Characterize The Landscape of Overparameterized
                  Convolutional Neural Networks},
  author =       {Yihong Gu and Weizhong Zhang and Cong Fang and J. Lee and Tong
                  Zhang},
  booktitle =    {NeurIPS},
  year =         2020,
  tags =         {neurips2020},
}

@misc{parkerholder2020ridge,
  title =        {Ridge Rider: Finding Diverse Solutions by Following
                  Eigenvectors of the Hessian},
  author =       {Jack Parker-Holder and Luke Metz and Cinjon Resnick and
                  Hengyuan Hu and Adam Lerer and Alistair Letcher and Alex
                  Peysakhovich and Aldo Pacchiano and Jakob Foerster},
  year =         2020,
  eprint =       {2011.06505},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {neurips2020},
}

@InProceedings{jastrzebski2021catastrophic,
  title =        {Catastrophic Fisher Explosion: Early Phase Fisher Matrix
                  Impacts Generalization},
  author =       {Jastrzebski, Stanislaw and Arpit, Devansh and Astrand, Oliver
                  and Kerg, Giancarlo B and Wang, Huan and Xiong, Caiming and
                  Socher, Richard and Cho, Kyunghyun and Geras, Krzysztof J},
  booktitle =    {Proceedings of the 38th International Conference on Machine
                  Learning},
  pages =        {4772--4784},
  year =         2021,
  editor =       {Meila, Marina and Zhang, Tong},
  volume =       139,
  series =       {Proceedings of Machine Learning Research},
  month =        {18--24 Jul},
  publisher =    {PMLR},
  pdf =
                  {http://proceedings.mlr.press/v139/jastrzebski21a/jastrzebski21a.pdf},
  url =          {https://proceedings.mlr.press/v139/jastrzebski21a.html},
}

@book{nielsen2010quantum,
  author =       {Nielsen, Michael A. and Chuang, Isaac L.},
  title =        {Quantum Computation and Quantum Information: 10th Anniversary
                  Edition},
  year =         2011,
  isbn =         1107002176,
  publisher =    {Cambridge University Press},
  address =      {USA},
  edition =      {10th},
  tags =         {skill},
}

@misc{nguyen2020tight,
  title =        {Tight Bounds on the Smallest Eigenvalue of the Neural Tangent
                  Kernel for Deep ReLU Networks},
  author =       {Quynh Nguyen and Marco Mondelli and Guido Montufar},
  year =         2020,
  eprint =       {2012.11654},
  archivePrefix ={arXiv},
  primaryClass = {stat.ML},
  tags =         {spectrum},
}

@misc{tselepidis2020twolevel,
  title =        {Two-Level K-FAC Preconditioning for Deep Learning},
  author =       {Nikolaos Tselepidis and Jonas Kohler and Antonio Orvieto},
  year =         2020,
  eprint =       {2011.00573},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {backpack, neurips2020},
}

@misc{lengyel2020genni,
  title =        {GENNI: Visualising the Geometry of Equivalences for Neural
                  Network Identifiability},
  author =       {Daniel Lengyel and Janith Petangoda and Isak Falk and Kate
                  Highnam and Michalis Lazarou and Arinbjörn Kolbeinsson and
                  Marc Peter Deisenroth and Nicholas R. Jennings},
  year =         2020,
  eprint =       {2011.07407},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {cockpit, neurips2020},
}

@misc{murfet2020deep,
  title =        {Deep Learning is Singular, and That's Good},
  author =       {Daniel Murfet and Susan Wei and Mingming Gong and Hui Li and
                  Jesse Gell-Redman and Thomas Quella},
  year =         2020,
  eprint =       {2010.11560},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {hessian},
}

@misc{agrawal2020investigating,
  title =        {Investigating Learning in Deep Neural Networks using
                  Layer-Wise Weight Change},
  author =       {Ayush Manish Agrawal and Atharva Tendle and Harshvardhan Sikka
                  and Sahib Singh and Amr Kayid},
  year =         2020,
  eprint =       {2011.06735},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {todo, cockpit},
}

@misc{sagun2017eigenvalues,
  title =        {Eigenvalues of the Hessian in Deep Learning: Singularity and
                  Beyond},
  author =       {Levent Sagun and Leon Bottou and Yann LeCun},
  year =         2017,
  tags =         {done, cockpit},
}

@misc{sagun2018empirical,
  title =        {Empirical Analysis of the Hessian of Over-Parametrized Neural
                  Networks},
  author =       {Levent Sagun and Utku Evci and V. Ugur Guney and Yann Dauphin
                  and Leon Bottou},
  year =         2018,
  tags =         {done, cockpit},
}

@misc{springenberg2015striving,
  title =        {Striving for Simplicity: The All Convolutional Net},
  author =       {Jost Tobias Springenberg and Alexey Dosovitskiy and Thomas
                  Brox and Martin Riedmiller},
  year =         2015,
  eprint =       {1412.6806},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {done, cockpit, backpack},
}

@misc{thompson2020computational,
  title =        {The Computational Limits of Deep Learning},
  author =       {Neil C. Thompson and Kristjan Greenewald and Keeheon Lee and
                  Gabriel F. Manso},
  year =         2020,
  eprint =       {2007.05558},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {done, cockpit},
}

@misc{rochette2019efficient,
  title =        {Efficient Per-Example Gradient Computations in Convolutional
                  Neural Networks},
  author =       {Gaspar Rochette and Andre Manoel and Eric W. Tramel},
  year =         2019,
  eprint =       {1912.06015},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {done, backpack},
}

@misc{meyer2020hutch,
  title =        {Hutch++: Optimal Stochastic Trace Estimation},
  author =       {Raphael A. Meyer and Cameron Musco and Christopher Musco and
                  David P. Woodruff},
  year =         2020,
}

@misc{wu2020dissecting,
  title =        {Dissecting Hessian: Understanding Common Structure of Hessian
                  in Neural Networks},
  author =       {Yikai Wu and Xingyu Zhu and Chenwei Wu and Annie Wang and Rong
                  Ge},
  year =         2020,
  tags =         {hbp},
}

@inproceedings{anonymous2021nngeometry,
  title =        {{\{}NNG{\}}eometry: Easy and Fast Fisher Information Matrices
                  and Neural Tangent Kernels in PyTorch},
  author =       {Anonymous},
  booktitle =    {Submitted to International Conference on Learning
                  Representations},
  year =         2021,
  url =          {https://openreview.net/forum?id=wabe-NE8-AX},
  note =         {under review},
  tags =         {todo, backpack},
}

@inproceedings{balles2017coupling,
  title =        {Coupling Adaptive Batch Sizes with Learning Rates},
  author =       {Balles, L. and Romero, J. and Hennig, P.},
  booktitle =    {Conference on Uncertainty in Artificial Intelligence (UAI)},
  year =         2017,
  tags =         {done, cockpit},
}

@misc{mahsereci2017early,
  title =        {Early Stopping without a Validation Set},
  author =       {Maren Mahsereci and Lukas Balles and Christoph Lassner and
                  Philipp Hennig},
  year =         2017,
  tags =         {done, cockpit},
}

@inproceedings{becigneul2018riemannian,
  title =        {Riemannian Adaptive Optimization Methods},
  author =       {Gary Becigneul and Octavian-Eugen Ganea},
  booktitle =    {International Conference on Learning Representations},
  year =         2019,
  url =          {https://openreview.net/forum?id=r1eiqi09K7},
  tags =         {skill, iclr2019},
}

@conference{schmidt2021descending,
  title =        {Descending through a Crowded Valley - Benchmarking Deep
                  Learning Optimizers},
  author =       {Schmidt, R. M. and Schneider, F. and Hennig, P.},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2021,
}

@software{bradbury2018jax,
  author = {James Bradbury and Roy Frostig and Peter Hawkins and Matthew James Johnson and Chris Leary and Dougal Maclaurin and Skye Wanderman-Milne},
  title = {{JAX}: composable transformations of {P}ython+{N}um{P}y programs},
  year = {2018},
}

@misc{panigrahi2019nongaussianity,
  title =        {Non-Gaussianity of Stochastic Gradient Noise},
  author =       {Abhishek Panigrahi and Raghav Somani and Navin Goyal and
                  Praneeth Netrapalli},
  year =         2019,
  eprint =       {1910.09626},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {todo, cockpit, neurips2019},
}

@misc{granziol2019deep,
  title =        {Deep Curvature Suite},
  author =       {Diego Granziol and Xingchen Wan and Timur Garipov},
  year =         2019,
  tags =         {done, backpack, hessian},
}

@article{akaike1974look,
  title =        {A new look at the statistical model identification},
  author =       {Akaike, Hirotugu},
  journal =      {IEEE transactions on automatic control},
  volume =       19,
  number =       6,
  pages =        {716--723},
  year =         1974,
  publisher =    {Ieee},
  tags =         {todo, skill},
}

@article{schmidt2014convergence,
  title =        {Convergence rate of stochastic gradient with constant step
                  size},
  author =       {Schmidt, Mark},
  year =         2014,
  pdf =
                  {https://www.cs.ubc.ca/~schmidtm/Documents/2014_Notes_ConstantStepSG.pdf},
  tags =         {todo, skill},
}

@inproceedings{thomas2020interplay,
  title =        {On the interplay between noise and curvature and its effect on
                  optimization and generalization},
  author =       {Thomas, Valentin and Pedregosa, Fabian and van Merri\"enboer,
                  Bart and Manzagol, Pierre-Antoine and Bengio, Yoshua and Roux,
                  Nicolas Le},
  booktitle =    {International Conference on Artificial Intelligence and
                  Statistics (AISTATS)},
  year =         2020,
}

@inproceedings{wang2020assessing,
  title =        {Assessing Local Generalization Capability in Deep Models},
  author =       {Wang, Huan and Keskar, Nitish Shirish and Xiong, Caiming and
                  Socher, Richard},
  booktitle =    {International Conference on Artificial Intelligence and
                  Statistics},
  pages =        {2077--2087},
  year =         2020,
  tags =         {done, aistats2020},
}

@InProceedings{liao2020automatic,
  title =        {Automatic Differentiation of Sketched Regression},
  author =       {Liao, Hang and Pearlmutter, Barak and Potluru, Vamsi and
                  Woodruff, David},
  pages =        {4367--4376},
  year =         2020,
  editor =       {Silvia Chiappa and Roberto Calandra},
  volume =       108,
  series =       {Proceedings of Machine Learning Research},
  address =      {Online},
  month =        {26--28 Aug},
  publisher =    {PMLR},
  pdf =          {http://proceedings.mlr.press/v108/liao20a/liao20a.pdf},
  url =          {http://proceedings.mlr.press/v108/liao20a.html},
  tags =         {done, aistats2020},
}

@misc{cai2020inversefree,
  title =        {An Inverse-free Truncated Rayleigh-Ritz Method for Sparse
                  Generalized Eigenvalue Problem},
  author =       {Yunfeng Cai and Ping Li},
  year =         2020,
  eprint =       {2003.10897},
  archivePrefix ={arXiv},
  primaryClass = {stat.ML},
  tags =         {done, aistats2020},
}

@misc{jiang2019accelerating,
  title =        {Accelerating Deep Learning by Focusing on the Biggest Losers},
  author =       {Angela H. Jiang and Daniel L. -K. Wong and Giulio Zhou and
                  David G. Andersen and Jeffrey Dean and Gregory R. Ganger and
                  Gauri Joshi and Michael Kaminksy and Michael Kozuch and
                  Zachary C. Lipton and Padmanabhan Pillai},
  year =         2019,
  eprint =       {1910.00762},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {done},
}

@misc{gabrielsson2019topology,
  title =        {A Topology Layer for Machine Learning},
  author =       {Rickard Brüel-Gabrielsson and Bradley J. Nelson and Anjan
                  Dwaraknath and Primoz Skraba and Leonidas J. Guibas and Gunnar
                  Carlsson},
  year =         2019,
  eprint =       {1905.12200},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {done, aistats2020},
}

@misc{jahani2018efficient,
  title =        {Efficient Distributed Hessian Free Algorithm for Large-scale
                  Empirical Risk Minimization via Accumulating Sample Strategy},
  author =       {Majid Jahani and Xi He and Chenxin Ma and Aryan Mokhtari and
                  Dheevatsa Mudigere and Alejandro Ribeiro and Martin Takáč},
  year =         2018,
  eprint =       {1810.11507},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {done, aistats2020},
}

@article{bollapragada2017adaptive,
  title =        {Adaptive Sampling Strategies for Stochastic Optimization},
  author =       {Raghu Bollapragada and Richard H. Byrd and Jorge Nocedal},
  journal =      {SIAM Journal on Optimization},
  year =         2017,
  volume =       28,
  tags =         {done, inner product test, orthogonality test, batch size
                  selection, cockpit},
}

@article{byrd2012sample,
  author =       {Byrd, Richard H. and Chin, Gillian M. and Nocedal, Jorge and
                  Wu, Yuchen},
  title =        {Sample Size Selection in Optimization Methods for Machine
                  Learning},
  year =         2012,
  volume =       134,
  journal =      {Math. Program.},
  tags =         {done, norm test, batch size selection, cockpit},
}

@article{fu2020waste,
  title =        {Don’t Waste Your Bits! Squeeze Activations and Gradients for
                  Deep Neural Networks via TINYSCRIPT},
  author =       {Fu, Fangcheng and Hu, Yuzheng and He, Yihan and Jiang, Jiawei
                  and Shao, Yingxia and Zhang, Ce and Cui, Bin},
  tags =         {todo},
}

@misc{anil2020second,
  title =        {Second Order Optimization Made Practical},
  author =       {Rohan Anil and Vineet Gupta and Tomer Koren and Kevin Regan
                  and Yoram Singer},
  year =         2020,
}

@book{petersen2015mastering,
  title =        {Mastering Emacs},
  author =       {Petersen, M.},
  isbn =         9781320673914,
  url =          {https://books.google.de/books?id=Gu7qsgEACAAJ},
  year =         2015,
  publisher =    {Blurb, Incorporated},
  tags =         {todo, fun, emacs},
}

@misc{chatterjee2020making,
  title =        {Making Coherence Out of Nothing At All: Measuring the
                  Evolution of Gradient Alignment},
  author =       {Satrajit Chatterjee and Piotr Zielinski},
  year =         2020,
  eprint =       {2008.01217},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {todo, cockpit, generalization},
}

@inproceedings{yao2020pyhessian,
  author =       {Yao, Zhewei and Gholami, Amir and Keutzer, Kurt and Mahoney,
                  Michael W.},
  booktitle =    {IEEE International Conference on Big Data},
  title =        {Py{H}essian: Neural Networks Through the Lens of the
                  {H}essian},
  year =         2020,
}

@misc{forouzesh2020generalization,
  title =        {Generalization Comparison of Deep Neural Networks via Output
                  Sensitivity},
  author =       {Mahsa Forouzesh and Farnood Salehi and Patrick Thiran},
  year =         2020,
  eprint =       {2007.15378},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {todo, generalization},
}

@misc{khan2019approximate,
  title =        {Approximate Inference Turns Deep Networks into Gaussian
                  Processes},
  author =       {Mohammad Emtiyaz Khan and Alexander Immer and Ehsan Abedi and
                  Maciej Korzepa},
  year =         2019,
  eprint =       {1906.01930},
  archivePrefix ={arXiv},
  primaryClass = {stat.ML},
  tags =         {todo, hessian, sameasimmer20disentangling},
}

@misc{immer2020disentangling,
  title =        {Disentangling the Gauss-Newton Method and Approximate
                  Inference for Neural Networks},
  author =       {Alexander Immer},
  year =         2020,
  eprint =       {2007.11994},
  archivePrefix ={arXiv},
  primaryClass = {stat.ML},
  tags =         {todo, hessian, sameaskhan201approximate},
}

@misc{tropp2015introduction,
  title =        {An Introduction to Matrix Concentration Inequalities},
  author =       {Joel A. Tropp},
  year =         2015,
  eprint =       {1501.01571},
  archivePrefix ={arXiv},
  primaryClass = {math.PR},
  tags =         {todo, fun},
}

@misc{dinh2017sharp,
  title =        {Sharp Minima Can Generalize For Deep Nets},
  author =       {Laurent Dinh and Razvan Pascanu and Samy Bengio and Yoshua
                  Bengio},
  year =         2017,
  eprint =       {1703.04933},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {done, hessian, generalization},
}

@misc{sankararaman2019impact,
  title =        {The Impact of Neural Network Overparameterization on Gradient
                  Confusion and Stochastic Gradient Descent},
  author =       {Karthik A. Sankararaman and Soham De and Zheng Xu and W. Ronny
                  Huang and Tom Goldstein},
  year =         2019,
  eprint =       {1904.06963},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {todo, generalization},
}

@article{ginsburg2020regularization,
  author =       {Boris Ginsburg},
  title =        {On regularization of gradient descent, layer imbalance and
                  flat minima},
  year =         2020,
  tags =         {done, hessian},
}

@misc{bahamou2019dynamic,
  title =        {A Dynamic Sampling Adaptive-SGD Method for Machine Learning},
  author =       {Achraf Bahamou and Donald Goldfarb},
  year =         2019,
  eprint =       {1912.13357},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {cockpit},
}

@inproceedings{ghorbani2019investigation,
  title =        {An Investigation into Neural Net Optimization via Hessian
                  Eigenvalue Density},
  author =       {Ghorbani, Behrooz and Krishnan, Shankar and Xiao, Ying},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2019,
  tags =         {todo, hessian},
}

@incollection{zhang2018local,
  title =        {On the Local Hessian in Back-propagation},
  author =       {Zhang, Huishuai and Chen, Wei and Liu, Tie-Yan},
  booktitle =    {Advances in Neural Information Processing Systems 31},
  editor =       {S. Bengio and H. Wallach and H. Larochelle and K. Grauman and
                  N. Cesa-Bianchi and R. Garnett},
  pages =        {6520--6530},
  year =         2018,
  publisher =    {Curran Associates, Inc.},
  url =
                  {http://papers.nips.cc/paper/7887-on-the-local-hessian-in-back-propagation.pdf},
  tags =         {todo, hessian},
}

@misc{jospin2020handson,
  title =        {Hands-on Bayesian Neural Networks -- a Tutorial for Deep
                  Learning Users},
  author =       {Laurent Valentin Jospin and Wray Buntine and Farid Boussaid
                  and Hamid Laga and Mohammed Bennamoun},
  year =         2020,
  eprint =       {2007.06823},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {todo, fun},
}

@article{shalev2010learnability,
  title =        {Learnability, stability and uniform convergence},
  author =       {Shalev-Shwartz, Shai and Shamir, Ohad and Srebro, Nathan and
                  Sridharan, Karthik},
  journal =      {The Journal of Machine Learning Research},
  volume =       11,
  pages =        {2635--2670},
  year =         2010,
  publisher =    {JMLR. org},
  tags =         {todo, cockpit},
}

@misc{nagarajan2019uniform,
  title =        {Uniform convergence may be unable to explain generalization in
                  deep learning},
  author =       {Vaishnavh Nagarajan and J. Zico Kolter},
  year =         2019,
  eprint =       {1902.04742},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {todo, cockpit},
}

@misc{fort2019stiffness,
  title =        {Stiffness: A New Perspective on Generalization in Neural
                  Networks},
  author =       {Stanislav Fort and Paweł Krzysztof Nowak and Stanislaw
                  Jastrzebski and Srini Narayanan},
  year =         2019,
  eprint =       {1901.09491},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {todo, cockpit},
}

@misc{frankle2020early,
  title =        {The Early Phase of Neural Network Training},
  author =       {Jonathan Frankle and David J. Schwab and Ari S. Morcos},
  year =         2020,
  eprint =       {2002.10365},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {cockpit},
}

@misc{faghri2020study,
  title =        {A Study of Gradient Variance in Deep Learning},
  author =       {Fartash Faghri and David Duvenaud and David J. Fleet and Jimmy
                  Ba},
  year =         2020,
  tags =         {cockpit},
}

@inproceedings{kunstner2019limitations,
  author =       {Kunstner, Frederik and Hennig, Philipp and Balles, Lukas},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  title =        {Limitations of the empirical Fisher approximation for natural
                  gradient descent},
  year =         2019,
  tags =         {done},
}

@article{livan2018introduction,
  title =        {Introduction to Random Matrices},
  ISBN =         9783319708850,
  ISSN =         {2197-1765},
  url =          {http://dx.doi.org/10.1007/978-3-319-70885-0},
  DOI =          {10.1007/978-3-319-70885-0},
  journal =      {SpringerBriefs in Mathematical Physics},
  publisher =    {Springer International Publishing},
  author =       {Livan, Giacomo and Novaes, Marcel and Vivo, Pierpaolo},
  year =         2018,
  tags =         {todo, fun},
}

@misc{pesme2020convergence,
  title =        {On Convergence-Diagnostic based Step Sizes for Stochastic
                  Gradient Descent},
  author =       {Scott Pesme and Aymeric Dieuleveut and Nicolas Flammarion},
  year =         2020,
  eprint =       {2007.00534},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {todo, cockpit},
}

@misc{sun2020global,
  title =        {The Global Landscape of Neural Networks: An Overview},
  author =       {Ruoyu Sun and Dawei Li and Shiyu Liang and Tian Ding and R
                  Srikant},
  year =         2020,
  eprint =       {2007.01429},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {todo, cockpit},
}

@inproceedings{zhang2020clipping,
  title =        {Why Gradient Clipping Accelerates Training: A Theoretical
                  Justification for Adaptivity},
  author =       {Jingzhao Zhang and Tianxing He and Suvrit Sra and Ali
                  Jadbabaie},
  booktitle =    {International Conference on Learning Representations},
  year =         2020,
  url =          {https://openreview.net/forum?id=BJgnXpVYwS},
  tags =         {todo, cockpit},
}

@inproceedings{chatterjee2020coherent,
  title =        {Coherent Gradients: An Approach to Understanding
                  Generalization in Gradient Descent-based Optimization},
  author =       {Satrajit Chatterjee},
  booktitle =    {International Conference on Learning Representations},
  year =         2020,
  url =          {https://openreview.net/forum?id=ryeFY0EFwS},
  tags =         {doing, cockpit},
}

@inproceedings{liu2020understanding,
  title =        {Understanding Why Neural Networks Generalize Well Through
                  {GSNR} of Parameters},
  author =       {Jinlong Liu and Yunzhi Bai and Guoqing Jiang and Ting Chen and
                  Huayan Wang},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2020,
  tags =         {done, cockpit},
}

@incollection{derezinski2019distributed,
  title =        {Distributed estimation of the inverse Hessian by determinantal
                  averaging},
  author =       {Derezinski, Michal and Mahoney, Michael W},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2019,
}

@incollection{edelman2013random,
  title =        {Random matrix theory and its innovative applications},
  author =       {Edelman, Alan and Wang, Yuyang},
  booktitle =    {Advances in Applied Mathematics, Modeling, and Computational
                  Science},
  pages =        {91--116},
  year =         2013,
  publisher =    {Springer},
  tags =         {todo},
}

@misc{adams2018estimating,
  title =        {Estimating the Spectral Density of Large Implicit Matrices},
  author =       {Ryan P. Adams and Jeffrey Pennington and Matthew J. Johnson
                  and Jamie Smith and Yaniv Ovadia and Brian Patton and James
                  Saunderson},
  year =         2018,
}

@inproceedings{jastrzebski2020break,
  title =        {The Break-Even Point on Optimization Trajectories of Deep
                  Neural Networks},
  author =       {Stanislaw Jastrzebski and Maciej Szymczak and Stanislav Fort
                  and Devansh Arpit and Jacek Tabor and Kyunghyun Cho* and
                  Krzysztof Geras*},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2020,
}

@misc{leclerc2020regimes,
  title =        {The Two Regimes of Deep Network Training},
  author =       {Guillaume Leclerc and Aleksander Madry},
  year =         2020,
  eprint =       {2002.10376},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {done, cockpit},
}

@misc{lewkowycz2020large,
  title =        {The large learning rate phase of deep learning: the catapult
                  mechanism},
  author =       {Aitor Lewkowycz and Yasaman Bahri and Ethan Dyer and Jascha
                  Sohl-Dickstein and Guy Gur-Ari},
  year =         2020,
  eprint =       {2003.02218},
  archivePrefix ={arXiv},
  primaryClass = {stat.ML},
  tags =         {done, cockpit},
}

@misc{arjevani2020second,
  title =        {Second-Order Information in Non-Convex Stochastic
                  Optimization: Power and Limitations},
  author =       {Yossi Arjevani and Yair Carmon and John C. Duchi and Dylan J.
                  Foster and Ayush Sekhari and Karthik Sridharan},
  year =         2020,
  eprint =       {2006.13476},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {todo},
}

@inproceedings{martens2010deep,
  author =       {Martens, James},
  year =         2010,
  title =        {Deep learning via {H}essian-free optimization},
  booktitle =    {International Conference on Machine Learning (ICML)},
  tags =         {done},
}

@misc{neklyudov2020involutive,
  title =        {Involutive MCMC: a Unifying Framework},
  author =       {Kirill Neklyudov and Max Welling and Evgenii Egorov and Dmitry
                  Vetrov},
  year =         2020,
  eprint =       {2006.16653},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {todo},
}

@book{magnus1999matrix,
  title =        {{M}atrix {D}ifferential {C}alculus with {A}pplications in
                  {S}tatistics and {E}conometrics},
  author =       {Magnus, J. R. and Neudecker, H.},
  series =       {Probabilistics and Statistics},
  year =         1999,
}

@misc{yang2020structured,
  title =        {Structured Stochastic Quasi-Newton Methods for Large-Scale
                  Optimization Problems},
  author =       {Minghan Yang and Dong Xu and Yongfeng Li and Zaiwen Wen and
                  Mengyun Chen},
  year =         2020,
  eprint =       {2006.09606},
  archivePrefix ={arXiv},
  primaryClass = {math.OC},
  tags =         {todo},
}

@misc{zhang2020stochastic,
  title =        {Stochastic Optimization with Non-stationary Noise},
  author =       {Jingzhao Zhang and Hongzhou Lin and Subhro Das and Suvrit Sra
                  and Ali Jadbabaie},
  year =         2020,
  eprint =       {2006.04429},
  archivePrefix ={arXiv},
  primaryClass = {math.OC},
  tags =         {done},
}

@inproceedings{dangel2020backpack,
  title =        {{B}ack{PACK}: Packing more into Backprop},
  author =       {Felix Dangel and Frederik Kunstner and Philipp Hennig},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2020,
  keywords =     {mywork},
  tags =         {done},
}

@InProceedings{dangel2020modular,
  title =        {Modular Block-diagonal Curvature Approximations for
                  Feedforward Architectures},
  author =       {Dangel, Felix and Harmeling, Stefan and Hennig, Philipp},
  booktitle =    {International Conference on Artificial Intelligence and
                  Statistics (AISTATS)},
  year =         2020,
  tags =         {done, aistats2020},
  keywords =     {mywork},
}

@InProceedings{wen2020empirical,
  title =        {An Empirical Study of Stochastic Gradient Descent with
                  Structured Covariance Noise},
  author =       {Wen, Yeming and Luk, Kevin and Gazeau, Maxime and Zhang,
                  Guodong and Chan, Harris and Ba, Jimmy},
  booktitle =    {International Conference on Artificial Intelligence and
                  Statistics (AISTATS)},
  year =         2020,
  tags =         {done, aistats2020},
}

@inproceedings{lorraine2020optimizing,
  title =        {Optimizing Millions of Hyperparameters by Implicit
                  Differentiation},
  author =       {Lorraine, Jonathan and Vicol, Paul and Duvenaud, David},
  booktitle =    {International Conference on Artificial Intelligence and
                  Statistics (AISTATS)},
  year =         2020,
  tags =         {done, aistats2020},
}

@InProceedings{kawaguchi2020ordered,
  title =        {Ordered SGD: A New Stochastic Optimization Framework for
                  Empirical Risk Minimization},
  author =       {Kawaguchi, Kenji and Lu, Haihao},
  booktitle =    {Proceedings of the Twenty Third International Conference on
                  Artificial Intelligence and Statistics},
  pages =        {669--679},
  year =         2020,
  editor =       {Chiappa, Silvia and Calandra, Roberto},
  volume =       108,
  series =       {Proceedings of Machine Learning Research},
  address =      {Online},
  month =        {26--28 Aug},
  publisher =    {PMLR},
  pdf =
                  {http://proceedings.mlr.press/v108/kawaguchi20a/kawaguchi20a.pdf},
  url =          {http://proceedings.mlr.press/v108/kawaguchi20a.html},
  tags =         {done, aistats2020},
}

@InProceedings{li2020understanding,
  title =        {Understanding Generalization in Deep Learning via Tensor
                  Methods},
  author =       {Li, Jingling and Sun, Yanchao and Su, Jiahao and Suzuki, Taiji
                  and Huang, Furong},
  booktitle =    {Proceedings of the Twenty Third International Conference on
                  Artificial Intelligence and Statistics},
  pages =        {504--515},
  year =         2020,
  editor =       {Chiappa, Silvia and Calandra, Roberto},
  volume =       108,
  series =       {Proceedings of Machine Learning Research},
  address =      {Online},
  month =        {26--28 Aug},
  publisher =    {PMLR},
  pdf =          {http://proceedings.mlr.press/v108/li20c/li20c.pdf},
  url =          {http://proceedings.mlr.press/v108/li20c.html},
  tags =         {todo, aistats2020},
}

@misc{gargiani2020promise,
  title =        {On the Promise of the Stochastic Generalized {G}auss-{N}ewton
                  Method for Training {DNN}s},
  author =       {Matilde Gargiani and Andrea Zanelli and Moritz Diehl and Frank
                  Hutter},
  year =         2020,
  tags =         {done},
}

@misc{granziol2020curvature,
  title =        {Curvature is Key: Sub-Sampled Loss Surfaces and the
                  Implications for Large Batch Training},
  author =       {Diego Granziol},
  year =         2020,
  eprint =       {2006.09092},
  archivePrefix ={arXiv},
  primaryClass = {stat.ML},
  tags =         {done, cockpit},
}

@misc{granziol2020flatness,
  title =        {Flatness is a False Friend},
  author =       {Diego Granziol},
  year =         2020,
  eprint =       {2006.09091},
  archivePrefix ={arXiv},
  primaryClass = {stat.ML},
  tags =         {done, cockpit},
}

@misc{cong2020hessian,
  title =        {GO Hessian for Expectation-Based Objectives},
  author =       {Yulai Cong and Miaoyun Zhao and Jianqiao Li and Junya Chen and
                  Lawrence Carin},
  year =         2020,
  eprint =       {2006.08873},
  archivePrefix ={arXiv},
  primaryClass = {stat.ML},
  tags =         {todo},
}

@inproceedings{li2018visualizing,
  title =        {Visualizing the loss landscape of neural nets},
  author =       {Li, Hao and Xu, Zheng and Taylor, Gavin and Studer, Christoph
                  and Goldstein, Tom},
  booktitle =    {Advances in Neural Information Processing Systems},
  pages =        {6389--6399},
  year =         2018,
  tags =         {todo, cockpit},
}

@misc{ishida2020do,
  title =        {Do We Need Zero Training Loss After Achieving Zero Training
                  Error?},
  author =       {Takashi Ishida and Ikko Yamane and Tomoya Sakai and Gang Niu
                  and Masashi Sugiyama},
  year =         2020,
  eprint =       {2002.08709},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {todo, cockpit},
}

@misc{sivaprasad2019optimizer,
  title =        {Optimizer Benchmarking Needs to Account for Hyperparameter
                  Tuning},
  author =       {Prabhu Teja Sivaprasad and Florian Mai and Thijs Vogels and
                  Martin Jaggi and François Fleuret},
  year =         2019,
  eprint =       {1910.11758},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
}

@misc{pilanci2020neural,
  title =        {Neural Networks are Convex Regularizers: Exact Polynomial-time
                  Convex Optimization Formulations for Two-Layer Networks},
  author =       {Mert Pilanci and Tolga Ergen},
  year =         2020,
  eprint =       {2002.10553},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {todo},
}

@misc{wu2019on,
  title =        {On the Noisy Gradient Descent that Generalizes as SGD},
  author =       {Jingfeng Wu and Wenqing Hu and Haoyi Xiong and Jun Huan and
                  Vladimir Braverman and Zhanxing Zhu},
  year =         2019,
  eprint =       {1906.07405},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {todo},
}

@misc{tran-dinh2020stochastic,
  title =        {Stochastic Gauss-Newton Algorithms for Nonconvex Compositional
                  Optimization},
  author =       {Quoc Tran-Dinh and Nhan H. Pham and Lam M. Nguyen},
  year =         2020,
  eprint =       {2002.07290},
  archivePrefix ={arXiv},
  primaryClass = {math.OC},
  tags =         {todo},
}

@inproceedings{mulayoff2020unique,
  title =        {Unique Properties of Flat Minima in Deep Networks},
  author =       {Mulayoff, Rotem and Michaeli, Tomer},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2020,
  tags =         {done, cockpit, neurips2020},
}

@InProceedings{katharopoulos18not,
  title =        {Not All Samples Are Created Equal: Deep Learning with
                  Importance Sampling},
  author =       {Katharopoulos, Angelos and Fleuret, Francois},
  booktitle =    {Proceedings of the 35th International Conference on Machine
                  Learning},
  pages =        {2525--2534},
  year =         2018,
  editor =       {Dy, Jennifer and Krause, Andreas},
  volume =       80,
  series =       {Proceedings of Machine Learning Research},
  address =      {Stockholmsmässan, Stockholm Sweden},
  month =        {10--15 Jul},
  publisher =    {PMLR},
  pdf =
                  {http://proceedings.mlr.press/v80/katharopoulos18a/katharopoulos18a.pdf},
  url =          {http://proceedings.mlr.press/v80/katharopoulos18a.html},
  abstract =     {Deep Neural Network training spends most of the computation on
                  examples that are properly handled, and could be ignored. We
                  propose to mitigate this phenomenon with a principled
                  importance sampling scheme that focuses computation on
                  "informative" examples, and reduces the variance of the
                  stochastic gradients during training. Our contribution is
                  twofold: first, we derive a tractable upper bound to the
                  per-sample gradient norm, and second we derive an estimator of
                  the variance reduction achieved with importance sampling,
                  which enables us to switch it on when it will result in an
                  actual speedup. The resulting scheme can be used by changing a
                  few lines of code in a standard SGD procedure, and we
                  demonstrate experimentally on image classification, CNN
                  fine-tuning, and RNN training, that for a fixed wall-clock
                  time budget, it provides a reduction of the train losses of up
                  to an order of magnitude and a relative improvement of test
                  errors between 5\% and 17\%.},
}

@misc{li2019tunefree,
  title =        {Almost Tune-Free Variance Reduction},
  author =       {Bingcong Li and Lingda Wang and Georgios B. Giannakis},
  year =         2019,
  eprint =       {1908.09345},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {todo},
}

@misc{bolte2020mathematical,
  title =        {A mathematical model for automatic differentiation in machine
                  learning},
  author =       {Jerome Bolte and Edouard Pauwels},
  year =         2020,
  eprint =       {2006.02080},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {done},
}

@misc{ly2017tutorial,
  title =        {A Tutorial on Fisher Information},
  author =       {Alexander Ly and Maarten Marsman and Josine Verhagen and Raoul
                  Grasman and Eric-Jan Wagenmakers},
  year =         2017,
  eprint =       {1705.01064},
  archivePrefix ={arXiv},
  primaryClass = {math.ST},
  tags =         {vivit},
}

@article{chen2020selftuning,
  title =        {Self-Tuning Stochastic Optimization with Curvature-Aware
                  Gradient Filtering},
  author =       {Chen, Ricky T. Q. and Choi, Dami and Balles, Lukas and
                  Duvenaud, David and Hennig, Philipp},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS),
                  Workshop I Can't Believe It's Not Better!},
  year =         2020
}

@misc{krishnan2017neumann,
  title =        {Neumann Optimizer: A Practical Optimization Algorithm for Deep
                  Neural Networks},
  author =       {Shankar Krishnan and Ying Xiao and Rif A. Saurous},
  year =         2017,
}

@article{amari2000natural,
  author =       {Amari, Shun-Ichi},
  year =         2000,
  title =        {Natural Gradient Works Efficiently in Learning},
  journal =      {Neural Computation},
  tags =         {vivit},
}

@inproceedings{singh2020woodfisher,
  author =       {Singh, Sidak Pal and Alistarh, Dan},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  title =        {WoodFisher: Efficient Second-Order Approximation for Neural
                  Network Compression},
  year =         2020,
  tags =         {vivit},
}

@misc{gressmann2020improving,
  title =        {Improving Neural Network Training in Low Dimensional Random
                  Bases},
  author =       {Frithjof Gressmann and Zach Eaton-Rosen and Carlo Luschi},
  year =         2020,
  eprint =       {2011.04720},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {vivit},
}

@misc{gurari2018gradient,
  title =        {Gradient Descent Happens in a Tiny Subspace},
  author =       {Guy Gur-Ari and Daniel A. Roberts and Ethan Dyer},
  year =         2018,
  tags =         {vivit},
}

@article{gratton2007approximate,
  title =        {Approximate Gauss--Newton methods for nonlinear least squares
                  problems},
  author =       {Gratton, Serge and Lawless, Amos S and Nichols, Nancy K},
  journal =      {SIAM Journal on Optimization},
  volume =       18,
  number =       1,
  pages =        {106--132},
  year =         2007,
  publisher =    {SIAM},
  tags =         {vivit},
}

@inproceedings{schneider2021cockpit,
  title =        {Cockpit: A Practical Debugging Tool for the Training of Deep
                  Neural Networks},
  author =       {Schneider, Frank and Dangel, Felix and Hennig, Philipp},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2021,
  keywords =     {mywork},
}

@misc{balestriero2021fast,
  title =        {Fast Jacobian-Vector Product for Deep Networks},
  author =       {Randall Balestriero and Richard Baraniuk},
  year =         2021,
  eprint =       {2104.00219},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG}
}

@book{heinrichs2013thank,
  title =        {Thank You For Arguing, Revised and Updated Edition: What
                  Aristotle, Lincoln, And Homer Simpson Can Teach Us About the
                  Art of Persuasion},
  author =       {Jay Heinrichs},
  isbn =         9780385347785,
  lccn =         2014378537,
  url =          {https://books.google.de/books?id=xzDKMNju-V4C},
  year =         2013,
  publisher =    {Crown/Archetype}
}

@inproceedings{dauphin2019metainit,
  author =       {Dauphin, Yann N and Schoenholz, Samuel},
  booktitle =    {Advances in Neural Information Processing Systems},
  editor =       {H. Wallach and H. Larochelle and A. Beygelzimer and F.
                  d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett},
  publisher =    {Curran Associates, Inc.},
  title =        {MetaInit: Initializing learning by learning to initialize},
  url =
                  {https://proceedings.neurips.cc/paper/2019/file/876e8108f87eb61877c6263228b67256-Paper.pdf},
  volume =       32,
  year =         2019,
  tags =         {neurips2019, hessian},
}

@misc{papyan2019spectrum,
  title =        {The Full Spectrum of Deepnet Hessians at Scale: Dynamics with
                  SGD Training and Sample Size},
  author =       {Vardan Papyan},
  year =         2019,
  eprint =       {1811.07062},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {hessian, vivit},
}

@article{corallo2021emacs,
  author =       {Andrea Corallo and Luca Nassi and Nicola Manca},
  title =        {Bringing {GNU} Emacs to Native Code},
  journal =      {CoRR},
  volume =       {abs/2004.02504},
  year =         2020,
  url =          {https://arxiv.org/abs/2004.02504},
  archivePrefix ={arXiv},
  eprint =       {2004.02504},
  timestamp =    {Wed, 08 Apr 2020 17:08:25 +0200},
  biburl =       {https://dblp.org/rec/journals/corr/abs-2004-02504.bib},
  bibsource =    {dblp computer science bibliography, https://dblp.org}
}

@misc{li2021low,
  title =        {Low Dimensional Landscape Hypothesis is True: DNNs can be
                  Trained in Tiny Subspaces},
  author =       {Tao Li and Lei Tan and Qinghua Tao and Yipeng Liu and Xiaolin
                  Huang},
  year =         2021,
  eprint =       {2103.11154},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG}
}

@misc{jorda2021cuconv,
  title =        {cuConv: A CUDA Implementation of Convolution for CNN
                  Inference},
  author =       {Marc Jordà and Pedro Valero-Lara and Antonio J. Peña},
  year =         2021,
  eprint =       {2103.16234},
  archivePrefix ={arXiv},
  primaryClass = {cs.DC},
  tags =         {backpack},
}

@incollection{paszke2019pytorch,
  title =        {{PyTorch}: An Imperative Style, High-Performance Deep Learning
                  Library},
  author =       {Paszke, Adam and Gross, Sam and Massa, Francisco and Lerer,
                  Adam and Bradbury, James and Chanan, Gregory and Killeen,
                  Trevor and Lin, Zeming and Gimelshein, Natalia and Antiga,
                  Luca and Desmaison, Alban and Kopf, Andreas and Yang, Edward
                  and DeVito, Zachary and Raison, Martin and Tejani, Alykhan and
                  Chilamkurthy, Sasank and Steiner, Benoit and Fang, Lu and Bai,
                  Junjie and Chintala, Soumith},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2019,
}

@misc{abadi2015tensorflow,
  title =        {{TensorFlow}: Large-Scale Machine Learning on Heterogeneous
                  Systems},
  author =       { Mart\'{\i}n Abadi and Ashish Agarwal and Paul Barham and
                  Eugene Brevdo and Zhifeng Chen and Craig Citro and Greg S.
                  Corrado and Andy Davis and Jeffrey Dean and Matthieu Devin and
                  Sanjay Ghemawat and Ian Goodfellow and Andrew Harp and
                  Geoffrey Irving and Michael Isard and Yangqing Jia and Rafal
                  Jozefowicz and Lukasz Kaiser and Manjunath Kudlur and Josh
                  Levenberg and Dan Man\'{e} and Rajat Monga and Sherry Moore
                  and Derek Murray and Chris Olah and Mike Schuster and Jonathon
                  Shlens and Benoit Steiner and Ilya Sutskever and Kunal Talwar
                  and Paul Tucker and Vincent Vanhoucke and Vijay Vasudevan and
                  Fernanda Vi\'{e}gas and Oriol Vinyals and Pete Warden and
                  Martin Wattenberg and Martin Wicke and Yuan Yu and Xiaoqiang
                  Zheng},
  year =         2015,
}

@inproceedings{osawa2019large,
  author =       {Osawa, Kazuki and Tsuji, Yohei and Ueno, Yuichiro and Naruse,
                  Akira and Yokota, Rio and Matsuoka, Satoshi},
  booktitle =    {2019 IEEE/CVF Conference on Computer Vision and Pattern
                  Recognition (CVPR)},
  title =        {Large-Scale Distributed Second-Order Optimization Using
                  Kronecker-Factored Approximate Curvature for Deep
                  Convolutional Neural Networks},
  year =         2019,
  pages =        {12351-12359},
  doi =          {10.1109/CVPR.2019.01264}
}

@InProceedings{botev2017practical,
  title =        {Practical {G}auss-{N}ewton Optimisation for Deep Learning},
  author =       {Aleksandar Botev and Hippolyt Ritter and David Barber},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2017,
}

@inproceedings{schneider2019deepobs,
  title =        {Deep{OBS}: A Deep Learning Optimizer Benchmark Suite},
  author =       {Schneider, Frank and Balles, Lukas and Hennig, Philipp},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2019,
}

@inproceedings{papyan2019measurements,
  author =       {Vardan Papyan},
  title =        {Measurements of Three-Level Hierarchical Structure in the
                  Outliers in the Spectrum of Deepnet {H}essians},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2019,
}

@article{cai2020gramgaussnewton,
  title =        {Gram-Gauss-Newton Method: Learning Overparameterized Neural
                  Networks for Regression Problems},
  author =       {Tianle Cai and Ruiqi Gao and Jikai Hou and Siyu Chen and Dong
                  Wang and Di He and Zhihua Zhang and Liwei Wang},
  year =         2020,
  url =          {https://openreview.net/forum?id=H1gCeyHFDS}
}

@article{chen2021fast,
  author =       {Chen, Chao and Reiz, Severin and Yu, Chenhan D. and Bungartz,
                  Hans-Joachim and Biros, George},
  title =        {Fast Approximation of the Gauss--Newton Hessian Matrix for the
                  Multilayer Perceptron},
  journal =      {SIAM Journal on Matrix Analysis and Applications (SIMAX)},
  year =         2021,
}

@article{papyan2020prevalence,
  title =        {Prevalence of neural collapse during the terminal phase of
                  deep learning training},
  journal =      {Proceedings of the National Academy of Sciences (PNAS)},
  author =       {Papyan, Vardan and Han, X. Y. and Donoho, David L.},
  year =         2020,
}

@article{papyan2020traces,
  author =       {Vardan Papyan},
  title =        {Traces of Class/Cross-Class Structure Pervade Deep Learning
                  Spectra},
  journal =      {Journal of Machine Learning Research (JMLR)},
  year =         2020,
}

@inproceedings{lecun1889optimal,
  author =       {LeCun, Yann and Denker, John and Solla, Sara},
  booktitle =    {Advances in Neural Information Processing Systems (NIPS)},
  title =        {Optimal Brain Damage},
  year =         1989
}

@misc{wei2020how,
  title =        {How noise affects the Hessian spectrum in overparameterized
                  neural networks},
  author =       {Mingwei Wei and David Schwab},
  year =         2020,
  url =          {https://openreview.net/forum?id=Hklcm0VYDS}
}

@inproceedings{bernacchia2018exact,
  author =       {Bernacchia, Alberto and Lengyel, Mate and Hennequin,
                  Guillaume},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  title =        {Exact natural gradient in deep linear networks and its
                  application to the nonlinear case},
  year =         2018,
  tags =         {vivit},
}

@book{kahneman2011thinking,
  author =       {Kahneman, Daniel},
  isbn =         {9780374275631 0374275637},
  publisher =    {Farrar, Straus and Giroux},
  refid =        706020998,
  title =        {Thinking, fast and slow},
  year =         2011
}

@article{dangel2021vivit,
  title =        {{ViViT}: {C}urvature access through the generalized
                  {G}auss-{N}ewton's low-rank structure},
  author =       {Felix Dangel and Lukas Tatzel and Philipp Hennig},
  year =         2021,
  primaryClass = {cs.LG},
  keywords =     {mywork},
}

@book{bluedorn2015fallacy,
  title =        {The Fallacy Detective: Thirty-Eight Lessons on How to
                  Recognize Bad Reasoning},
  author =       {Bluedorn, N. and Bluedorn, H. and Corley, R.},
  isbn =         9780974531595,
  lccn =         2013944061,
  url =          {https://books.google.de/books?id=Uvs7xQEACAAJ},
  year =         2015,
  publisher =    {Christian Logic}
}

@InProceedings{finn2017model,
  title =        {Model-Agnostic Meta-Learning for Fast Adaptation of Deep
                  Networks},
  author =       {Chelsea Finn and Pieter Abbeel and Sergey Levine},
  booktitle =    {Proceedings of the 34th International Conference on Machine
                  Learning},
  pages =        {1126--1135},
  year =         2017,
  editor =       {Precup, Doina and Teh, Yee Whye},
  volume =       70,
  series =       {Proceedings of Machine Learning Research},
  month =        {06--11 Aug},
  publisher =    {PMLR},
  pdf =          {http://proceedings.mlr.press/v70/finn17a/finn17a.pdf},
  url =          { http://proceedings.mlr.press/v70/finn17a.html},
  tags =         {skill},
}

@misc{frankle2019lottery,
  title =        {The Lottery Ticket Hypothesis: Finding Sparse, Trainable
                  Neural Networks},
  author =       {Jonathan Frankle and Michael Carbin},
  year =         2019,
  eprint =       {1803.03635},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG}
}

@misc{jacot2020neural,
  title =        {Neural Tangent Kernel: Convergence and Generalization in
                  Neural Networks},
  author =       {Arthur Jacot and Franck Gabriel and Clément Hongler},
  year =         2020,
  eprint =       {1806.07572},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG},
  tags =         {todo},
}

@misc{xu2018secondorder,
  title =        {Second-Order Optimization for Non-Convex Machine Learning: An
                  Empirical Study},
  author =       {Peng Xu and Farbod Roosta-Khorasani and Michael W. Mahoney},
  year =         2018,
  eprint =       {1708.07827},
  archivePrefix ={arXiv},
  primaryClass = {math.OC}
}

@misc{ortizjiménez2021linearized,
  title =        {What can linearized neural networks actually say about
                  generalization?},
  author =       {Guillermo Ortiz-Jiménez and Seyed-Mohsen Moosavi-Dezfooli and
                  Pascal Frossard},
  year =         2021,
  eprint =       {2106.06770},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG}
}

@misc{papamakarios2021normalizing,
  title =        {Normalizing Flows for Probabilistic Modeling and Inference},
  author =       {George Papamakarios and Eric Nalisnick and Danilo Jimenez
                  Rezende and Shakir Mohamed and Balaji Lakshminarayanan},
  year =         2021,
  eprint =       {1912.02762},
  archivePrefix ={arXiv},
  primaryClass = {stat.ML}
}

@misc{ren2019efficient,
  title =        {Efficient Subsampled Gauss-Newton and Natural Gradient Methods
                  for Training Neural Networks},
  author =       {Yi Ren and Donald Goldfarb},
  year =         2019,
  eprint =       {1906.02353},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG}
}

@misc{zhang2019fixup,
  title =        {Fixup Initialization: Residual Learning Without Normalization},
  author =       {Hongyi Zhang and Yann N. Dauphin and Tengyu Ma},
  year =         2019,
  eprint =       {1901.09321},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG}
}

@book{nocedal2006numerical,
  Title =        {Numerical Optimization},
  Author =       {Jorge Nocedal and Stephen J. Wright},
  Publisher =    {Springer},
  Year =         2006,
}

@misc{george2018fast,
  title =        {Fast Approximate Natural Gradient Descent in a
                  Kronecker-factored Eigenbasis},
  author =       {Thomas George and César Laurent and Xavier Bouthillier and
                  Nicolas Ballas and Pascal Vincent},
  year =         2018,
  eprint =       {1806.03884},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG}
}

@inproceedings{daxberger2021bayesian,
  title =        {Bayesian Deep Learning via Subnetwork Inference},
  author =       {Daxberger, E. and Nalisnick, E. and Allingham, J. and Antorán,
                  J. and Hernández-Lobato, J. M.},
  booktitle =    {38th International Conference on Machine Learning},
  month =        jul,
  year =         2021,
  month_numeric =7
}

@book{sarkka2013bayesian,
  title =        {Bayesian Filtering and Smoothing},
  author =       {Simo S{\"a}rkk{\"a}},
  year =         2013,
  isbn =         9781107619289,
  publisher =    {Cambridge University Press},
  address =      {United Kingdom},
  url =
                  {https://users.aalto.fi/~ssarkka/pub/cup_book_online_20131111.pdf},
}

@inproceedings{huh2020curvature,
  title =        {Curvature-corrected learning dynamics in deep neural networks},
  author =       {Huh, Dongsung},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2020,
}

@misc{saxe2014exact,
  title =        {Exact solutions to the nonlinear dynamics of learning in deep
                  linear neural networks},
  author =       {Andrew M. Saxe and James L. McClelland and Surya Ganguli},
  year =         2014,
  eprint =       {1312.6120},
  archivePrefix ={arXiv},
  primaryClass = {cs.NE}
}

@inproceedings{welling2011bayesian,
  author =       {Welling, Max and Teh, Yee Whye},
  title =        {Bayesian Learning via Stochastic Gradient Langevin Dynamics},
  year =         2011,
  isbn =         9781450306195,
  publisher =    {Omnipress},
  address =      {Madison, WI, USA},
  booktitle =    {Proceedings of the 28th International Conference on
                  International Conference on Machine Learning},
  pages =        {681–688},
  numpages =     8,
  location =     {Bellevue, Washington, USA},
  series =       {ICML'11}
}

@misc{kao2021natural,
  title =        {Natural continual learning: success is a journey, not (just) a
                  destination},
  author =       {Ta-Chu Kao and Kristopher T. Jensen and Alberto Bernacchia and
                  Guillaume Hennequin},
  year =         2021,
  eprint =       {2106.08085},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG}
}

@inproceedings{vinyals2012krylov,
  title =        {Krylov Subspace Descent for Deep Learning},
  author =       {Oriol Vinyals and Daniel Povey},
  booktitle =    {Proceedings of the Fifteenth International Conference on
                  Artificial Intelligence and Statistics},
  pages =        {1261--1268},
  year =         2012,
  editor =       {Neil D. Lawrence and Mark Girolami},
  volume =       22,
  series =       {Proceedings of Machine Learning Research},
  address =      {La Palma, Canary Islands},
  month =        {21--23 Apr},
  publisher =    {PMLR},
  pdf =          {http://proceedings.mlr.press/v22/vinyals12/vinyals12.pdf},
  url =          {http://proceedings.mlr.press/v22/vinyals12.html},
}

@misc{song2018accelerating,
  title =        {Accelerating Natural Gradient with Higher-Order Invariance},
  author =       {Yang Song and Jiaming Song and Stefano Ermon},
  year =         2018,
  eprint =       {1803.01273},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG}
}

@misc{zhang2017blockdiagonal,
  title =        {Block-diagonal {H}essian-free Optimization for Training Neural
                  Networks},
  author =       {Huishuai Zhang and Caiming Xiong and James Bradbury and
                  Richard Socher},
  year =         2017,
}

@misc{hooker2020hardware,
  title =        {The Hardware Lottery},
  author =       {Sara Hooker},
  year =         2020,
  eprint =       {2009.06489},
  archivePrefix ={arXiv},
  primaryClass = {cs.CY}
}

@misc{hasse2020exercise,
  title =        {Exercise Book Improved Reading (workshop)},
  author =       {Friedrich Hasse},
  year =         2020,
  url =
                  {https://webcoachedtraining.de/app/content/shared/pdf/workbook_en.pdf}
}

@misc{hasse2020course,
  title =        {Couse Book Book Improved Reading (workshop)},
  author =       {Friedrich Hasse},
  year =         2020,
  url =
                  {https://webcoachedtraining.de/app/content/shared/pdf/workbook_en.pdf}
}

@misc{mishchenko2021regularized,
  title =        {Regularized Newton Method with Global O(1/k2) Convergence},
  author =       {Konstantin Mishchenko},
  year =         2021,
}

@book{pan2019sorry,
  title =        {Sorry I'm Late, I Didn't Want to Come: An Introvert’s Year of
                  Living Dangerously},
  author =       {Pan, J.},
  isbn =         9781473562707,
  url =          {https://books.google.de/books?id=VCJsDwAAQBAJ},
  year =         2019,
  publisher =    {Transworld}
}

@inproceedings{ioffe2015batch,
  title =        {Batch Normalization: Accelerating Deep Network Training by
                  Reducing Internal Covariate Shift},
  author =       {Ioffe, Sergey and Szegedy, Christian},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2015,
}

@misc{khan2021bayesian,
  title =        {The Bayesian Learning Rule},
  author =       {Mohammad Emtiyaz Khan and Håvard Rue},
  year =         2021,
}

@misc{smith2021origin,
  title =        {On the Origin of Implicit Regularization in Stochastic
                  Gradient Descent},
  author =       {Samuel L. Smith and Benoit Dherin and David G. T. Barrett and
                  Soham De},
  year =         2021,
  eprint =       {2101.12176},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG}
}

@misc{lin2021tractable,
  title =        {Tractable structured natural gradient descent using local
                  parameterizations},
  author =       {Wu Lin and Frank Nielsen and Mohammad Emtiyaz Khan and Mark
                  Schmidt},
  year =         2021,
  eprint =       {2102.07405},
  archivePrefix ={arXiv},
  primaryClass = {stat.ML}
}

@inproceedings{daxberger2021laplace,
  title =        {Laplace Redux - Effortless Bayesian Deep Learning},
  author =       {Erik Daxberger and Agustinus Kristiadi and Alexander Immer and
                  Runa Eschenhagen and Matthias Bauer and Philipp Hennig},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2021,
}

@inproceedings{martens2018kroneckerfactored,
  title =        {Kronecker-factored Curvature Approximations for Recurrent
                  Neural Networks},
  author =       {James Martens and Jimmy Ba and Matt Johnson},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2018,
}

@article{deroos2017krylov,
  title =        {Krylov Subspace Recycling for Fast Iterative Least-Squares in
                  Machine Learning},
  author =       {de Roos, Filip and Hennig, Philipp},
  journal =      {arXiv preprint arXiv:1706.00241},
  year =         2017,
  url =          {https://arxiv.org/abs/1706.00241}
}

@inproceedings{sohldickstein2014fast,
  title =        {Fast large-scale optimization by unifying stochastic gradient
                  and quasi-Newton methods},
  author =       {Sohl-Dickstein, Jascha and Poole, Ben and Ganguli, Surya},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2014,
}

@misc{tuddenham2020quasinewtons,
  title =        {Quasi-Newton's method in the class gradient defined
                  high-curvature subspace},
  author =       {Mark Tuddenham and Adam Prügel-Bennett and Jonathan Hare},
  year =         2020,
  eprint =       {2012.01938},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG}
}

@article{ubaru2017fast,
  title =        {Fast Estimation of tr(f(A)) via Stochastic Lanczos Quadrature},
  author =       {Shashanka Ubaru and Jie Chen and Yousef Saad},
  journal =      {SIAM J. Matrix Anal. Appl.},
  year =         2017,
  volume =       38,
  pages =        {1075-1099}
}

@article{lin2013approximating,
  author =       {Lin, Lin and Saad, Yousef and Yang, Chao},
  year =         2013,
  month =        08,
  title =        {Approximating Spectral Densities of Large Matrices},
  volume =       58,
  journal =      {SIAM Review},
  doi =          {10.1137/130934283}
}

@misc{amos2017input,
  title =        {Input Convex Neural Networks},
  author =       {Brandon Amos and Lei Xu and J. Zico Kolter},
  year =         2017,
  eprint =       {1609.07152},
  archivePrefix ={arXiv},
  primaryClass = {cs.LG}
}

@article{zhou2021damped,
  author =       {Zhou, Jingcheng and Wei, Wei and Zhang, Ruizhi and Zheng,
                  Zhiming},
  title =        {Damped Newton Stochastic Gradient Descent Method for Neural
                  Networks Training},
  journal =      {Mathematics},
  volume =       9,
  year =         2021,
  number =       13,
  article-number =1533,
  url =          {https://www.mdpi.com/2227-7390/9/13/1533},
  issn =         {2227-7390},
  doi =          {10.3390/math9131533}
}

@inproceedings{liu2021learning,
  title =        {Learning by Turning: Neural Architecture Aware Optimisation},
  author =       {Yang Liu and Jeremy Bernstein and Markus Meister and Yisong
                  Yue},
  year =         2021,
  eprint =       {2102.07227},
  archivePrefix ={arXiv},
  primaryClass = {cs.NE},
  tags =         {icml2021}
}

@article{nicholas2011quick,
  author =       {Nicholas, Kimberly A. and Gordon, Wendy S.},
  title =        {A quick guide to writing a solid peer review},
  journal =      {Eos, Transactions American Geophysical Union},
  volume =       92,
  number =       28,
  pages =        {233-234},
  keywords =     {peer review, professional development, scientific skills,
                  graduate training},
  doi =          {https://doi.org/10.1029/2011EO280001},
  url =
                  {https://agupubs.onlinelibrary.wiley.com/doi/abs/10.1029/2011EO280001},
  eprint =
                  {https://agupubs.onlinelibrary.wiley.com/doi/pdf/10.1029/2011EO280001},
  year =         2011
}

@misc{sankaran2022benchmarking,
  title =        {Benchmarking the Linear Algebra Awareness of {T}ensor{F}low
                  and {P}y{T}orch},
  author =       {Aravind Sankaran and Navid Akbari Alashti and Christos Psarras
                  and Paolo Bientinesi},
  year =         2022,
}

@article{hughes1989functional,
  AUTHOR =       {J. Hughes},
  TITLE =        {{Why Functional Programming Matters}},
  JOURNAL =      {Computer Journal},
  VOLUME =       32,
  NUMBER =       2,
  PAGES =        {98--107},
  YEAR =         1989
}

@article{amid2021locoprop,
  title =        {Locoprop: Enhancing backprop via local loss optimization},
  author =       {Amid, Ehsan and Anil, Rohan and Warmuth, Manfred K},
  journal =      {arXiv preprint arXiv:2106.06199},
  year =         2021
}

@article{bahamou2022mini,
  title =        {A Mini-Block Natural Gradient Method for Deep Neural Networks},
  author =       {Bahamou, Achraf and Goldfarb, Donald and Ren, Yi},
  journal =      {arXiv preprint arXiv:2202.04124},
  year =         2022
}

@inproceedings{hayashi2019einconv,
  author =       {Hayashi, Kohei and Yamaguchi, Taiki and Sugawara, Yohei and
                  Maeda, Shin-ichi},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  title =        {Exploring Unexplored Tensor Network Decompositions for
                  Convolutional Neural Networks},
  year =         2019
}

@article{marklin2001effect,
  author =       {Marklin, Richard W and Simoneau, Guy G},
  title =        "{Effect of Setup Configurations of Split Computer Keyboards on
                  Wrist Angle}",
  journal =      {Physical Therapy},
  volume =       81,
  number =       4,
  pages =        {1038-1048},
  year =         2001,
  month =        04,
  issn =         {0031-9023},
  doi =          {10.1093/ptj/81.4.1038},
  url =          {https://doi.org/10.1093/ptj/81.4.1038},
  eprint =
                  {https://academic.oup.com/ptj/article-pdf/81/4/1038/31683805/ptj1038.pdf},
}

@inproceedings{paszke2021getting,
  title =        {Getting to the Point. Index Sets and Parallelism-Preserving
                  Autodiff for Pointful Array Programming},
  author =       {Paszke, Adam and Johnson, Daniel and Duvenaud, David and
                  Vytiniotis, Dimitrios and Radul, Alexey and Johnson, Matthew
                  and Ragan-Kelley, Jonathan and Maclaurin, Dougal},
  booktitle =    {International Conference on Functional Programming},
  year =         2021
}

@inproceedings{oktay2021randomized,
  title =        {Randomized Automatic Differentiation},
  author =       {Deniz Oktay and Nick McGreivy and Joshua Aduol and Alex
                  Beatson and Ryan P Adams},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2021,
}

@misc{yang2020sketchy,
  doi =          {10.48550/ARXIV.2006.05924},
  url =          {https://arxiv.org/abs/2006.05924},
  author =       {Yang, Minghan and Xu, Dong and Wen, Zaiwen and Chen, Mengyun
                  and Xu, Pengxiang},
  keywords =     {Optimization and Control (math.OC), Machine Learning
                  (stat.ML), FOS: Mathematics, FOS: Mathematics, FOS: Computer
                  and information sciences, FOS: Computer and information
                  sciences},
  title =        {Sketchy Empirical Natural Gradient Methods for Deep Learning},
  publisher =    {arXiv},
  year =         2020,
  copyright =    {arXiv.org perpetual, non-exclusive license}
}

@book{clark2011fit,
  title =        {Fit ohne Ger{\"a}te: Trainieren mit dem eigenen
                  K{\"o}rpergewicht},
  author =       {Clark, J. and Lauren, M.},
  isbn =         9783864131523,
  url =          {https://books.google.de/books?id=XUgWuwSVI0wC},
  year =         2011,
  publisher =    {Riva}
}

@misc{zhang2022stack,
  doi =          {10.48550/ARXIV.2203.16338},
  url =          {https://arxiv.org/abs/2203.16338},
  author =       {Zhang, Tianning and Ang, L. K. and Chen, Tianqi and Yang, Bo
                  and Li, Erping},
  title =        {Stack operation of tensor networks},
  publisher =    {arXiv},
  year =         2022,
  copyright =    {Creative Commons Attribution 4.0 International}
}

@article{bottou2016machine,
  author =       {Bottou, Léon and Curtis, Frank E. and Nocedal, Jorge},
  title =        {Optimization Methods for Large-Scale Machine Learning},
  volume =       60,
  journal =      {SIAM Review (SIREV)},
  year =         2016,
}

@misc{frostig2021decomposing,
  doi =          {10.48550/ARXIV.2105.09469},
  url =          {https://arxiv.org/abs/2105.09469},
  author =       {Frostig, Roy and Johnson, Matthew J. and Maclaurin, Dougal and
                  Paszke, Adam and Radul, Alexey},
  keywords =     {Programming Languages (cs.PL), Machine Learning (cs.LG), FOS:
                  Computer and information sciences, FOS: Computer and
                  information sciences},
  title =        {Decomposing reverse-mode automatic differentiation},
  publisher =    {arXiv},
  year =         2021,
  copyright =    {arXiv.org perpetual, non-exclusive license}
}

@misc{radul2022you,
  author =       {Radul, Alexey and Paszke, Adam and Frostig, Roy and Johnson,
                  Matthew and Maclaurin, Dougal},
  title =        {You Only Linearize Once: Tangents Transpose to Gradients},
  year =         2022,
}

@incollection{rumelhart1986learning,
  author =       {Rumelhart, David E. and Hinton, Geoffrey E. and Williams,
                  Ronald J.},
  booktitle =    {Parallel Distributed Processing: Explorations in the
                  Microstructure of Cognition},
  title =        {Learning Internal Representations by Error Propagation},
  year =         1986
}

@article{chen2018bdapch,
  title =        {{BDA-PCH}: Block-Diagonal Approximation of Positive-Curvature
                  {H}essian for Training Neural Networks},
  author =       {Chen, Sheng-Wei and Chou, Chun-Nan and Chang, Edward},
  year =         2018
}

@inproceedings{becker1989improving,
  author =       {Becker, Suzanna and Lecun, Yann},
  year =         1989,
  title =        {Improving the Convergence of Back-Propagation Learning with
                  Second-Order Methods}
}

@software{novik2020torchoptimizer,
    title        = {{torch-optimizer -- collection of optimization algorithms for {P}y{T}orch}},
    author       = {Novik, Mykola},
    year         = 2020,
}

@software{he2021functorch,
  author =       {Horace He, Richard Zou},
  title =        {functorch: {JAX}-like composable function transforms for {P}y{T}orch},
  year =         {2021}
}

@misc{shao2022tensor,
  author =       {Shao, Junru and Zhou, Xiyou and Feng, Siyuan and Hou, Bohan
                  and Lai, Ruihang and Jin, Hongyi and Lin, Wuwei and Masuda,
                  Masahiro and Yu, Cody Hao and Chen, Tianqi},
  title =        {Tensor Program Optimization with Probabilistic Programs},
  year =         2022,
}

@article{smith2018opteinsum,
  author =       {Daniel G. A. Smith and Johnnie Gray},
  title =        {opt{\_}einsum - {A} Python package for optimizing contraction
                  order for einsum-like expressions},
  journal =      {Journal of Open Source Software (JOSS)},
  year =         2018,
}

@inproceedings{paszke2017automatic,
  author =       {Paszke, Adam and Gross, Sam and Chintala, Soumith and Chanan,
                  Gregory and Yang, Edward and DeVito, Zachary and Lin, Zeming
                  and Desmaison, Alban and Antiga, Luca and Lerer, Adam},
  booktitle =    {NIPS Workshop on Autodiff},
  title =        {Automatic Differentiation in {P}y{T}orch},
  year =         2017
}

@inproceedings{rame2022fishr,
  title =        {Fishr: Invariant Gradient Variances for Out-of-distribution
                  Generalization},
  author =       {Alexandre Rame and Corentin Dancette and Matthieu Cord},
  year =         2022,
  booktitle =    {International Conference on Machine Learning (ICML)}
}

@inproceedings{immer2021improving,
  title =        { Improving predictions of {B}ayesian neural nets via local
                  linearization },
  author =       {Immer, Alexander and Korzepa, Maciej and Bauer, Matthias},
  booktitle =    {International Conference on Artificial Intelligence and
                  Statistics (AISTATS)},
  year =         2021,
}

@inproceedings{gulrajani2021in,
  title =        {In Search of Lost Domain Generalization},
  author =       {Ishaan Gulrajani and David Lopez-Paz},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2021,
}

@inproceedings{yousefpour2021opacus,
  title =        {Opacus: User-Friendly Differential Privacy Library in PyTorch},
  author =       {Ashkan Yousefpour and Igor Shilov and Alexandre Sablayrolles
                  and Davide Testuggine and Karthik Prasad and Mani Malek and
                  John Nguyen and Sayan Ghosh and Akash Bharadwaj and Jessica
                  Zhao and Graham Cormode and Ilya Mironov},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS),
                  Workshop Privacy in Machine Learning},
  year =         2021,
}

@inproceedings{chen2019neural,
  title =        {Neural Networks with Cheap Differential Operators},
  author =       {Chen, Ricky T. Q. and Duvenaud, David},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2019,
}

@software{maclaurin2015autograd,
  author = {Dougal Maclaurin and David Duvenaud and Matthew Johnson and Ryan P. Adams},
  title = {Autograd: Reverse-mode differentiation of native {P}ython},
  year = {2015},
}

@article {maclaurin2019dex,
  title =        {Dex: array programming with typed indices},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS),
                  Workshop Program Transformations},
  year =         2019,
  author =       {Dougal Maclaurin and Radul, Alexey and Johnson, Matthew J and
                  Vytiniotis, Dimitrios}
}

@inproceedings{zhang2019fast,
  author =       {Zhang, Guodong and Martens, James and Grosse, Roger B},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  title =        {Fast Convergence of Natural Gradient Descent for
                  Over-Parameterized Neural Networks},
  year =         2019
}

@article{loan2000ubiquitous,
  title =        {The ubiquitous {K}ronecker product},
  journal =      {Journal of Computational and Applied Mathematics},
  year =         2000,
  author =       {Charles F.Van Loan},
}

@conference{balles2018dissecting,
  title =        {Dissecting Adam: The Sign, Magnitude and Variance of
                  Stochastic Gradients},
  author =       {Balles, Lukas and Hennig, Philipp},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2018,
}

@inproceedings{hassibi1992second,
  author =       {Hassibi, Babak and Stork, David},
  booktitle =    {Advances in Neural Information Processing Systems (NIPS)},
  title =        {Second order derivatives for network pruning: Optimal Brain
                  Surgeon},
  year =         1992
}

@inproceedings{zhao2015stochastic,
  title =        {Stochastic Optimization with Importance Sampling for
                  Regularized Loss Minimization},
  author =       {Zhao, Peilin and Zhang, Tong},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2015,
}

@inproceedings{needell2014stochastic,
  author =       {Needell, Deanna and Ward, Rachel and Srebro, Nati},
  booktitle =    {Advances in Neural Information Processing Systems (NIPS)},
  title =        {Stochastic Gradient Descent, Weighted Sampling, and the
                  Randomized {K}aczmarz algorithm},
  year =         2014
}

@article{wang2017accelerating,
  title =        {Accelerating Deep Neural Network Training with Inconsistent
                  Stochastic Gradient Descent},
  author =       {Linnan Wang and Yi Yang and Martin Renqiang Min and Srimat T.
                  Chakradhar},
  journal =      {Neural networks},
  year =         2017,
}

@inproceedings{abadi2016deep,
  author =       {Abadi, Martin and Chu, Andy and Goodfellow, Ian and McMahan,
                  H. Brendan and Mironov, Ilya and Talwar, Kunal and Zhang, Li},
  title =        {Deep Learning with Differential Privacy},
  year =         2016,
  booktitle =    {ACM SIGSAC Conference on Computer and Communications Security},
}

@inproceedings{fredrikson2015model,
  title =        {Model Inversion Attacks That Exploit Confidence Information
                  and Basic Countermeasures},
  year =         2015,
  author =       {Fredrikson, Matt and Jha, Somesh and Ristenpart, Thomas},
  booktitle =    {ACM Conference on Computer and Communications Security (CCS)}
}

@inproceedings{shokri2015privacy,
  author =       {Shokri, Reza and Shmatikov, Vitaly},
  title =        {Privacy-Preserving Deep Learning},
  year =         2015,
  booktitle =    {ACM SIGSAC Conference on Computer and Communications Security},
}

@article{robbins1951stochastic,
  author =       {Herbert Robbins and Sutton Monro},
  title =        {{A Stochastic Approximation Method}},
  journal =      {The Annals of Mathematical Statistics},
  year =         1951,
}

@article{polyak1964some,
  title =        {Some methods of speeding up the convergence of iteration
                  methods},
  journal =      {USSR Computational Mathematics and Mathematical Physics},
  year =         1964,
  author =       {B.T. Polyak},
}

@article{nesterov1983method,
  title =        {A method for solving the convex programming problem with
                  convergence rate $O(1/k^2)$},
  author =       {Yurii Nesterov},
  journal =      {Proceedings of the USSR Academy of Sciences},
  year =         1983,
}

@article{duchi2011adaptive,
  author =       {John Duchi and Elad Hazan and Yoram Singer},
  title =        {Adaptive Subgradient Methods for Online Learning and
                  Stochastic Optimization},
  journal =      {Journal of Machine Learning Research (JMLR)},
  year =         2011,
}

@article{tieleman2012lecture,
  title =        {Lecture 6.5-rmsprop: Divide the gradient by a running average
                  of its recent magnitude},
  author =       {Tieleman, Tijmen and Hinton, Geoffrey and others},
  journal =      {COURSERA: Neural networks for machine learning},
  year =         2012
}

@article{zeiler2012adadelta,
  author =       {Zeiler, Matthew},
  year =         2012,
  title =        {ADADELTA: An adaptive learning rate method},
}

@inproceedings{kingma2015adam,
  title =        {{A}dam: A Method for Stochastic Optimization},
  author =       {Kingma, Diederik P and Ba, Jimmy},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2015
}

@inproceedings{reddi2018on,
  title =        {On the Convergence of Adam and Beyond},
  author =       {Sashank J. Reddi and Satyen Kale and Sanjiv Kumar},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2018,
}

@misc{choi2020on,
  title =        {On Empirical Comparisons of Optimizers for Deep Learning},
  author =       {Dami Choi and Christopher J. Shallue and Zachary Nado and
                  Jaehoon Lee and Chris J. Maddison and George E. Dahl},
  year =         2020,
}

@article{rosenblatt1958perceptron,
  title =        {The perceptron: a probabilistic model for information storage
                  and organization in the brain.},
  author =       {Frank Rosenblatt},
  journal =      {Psychological Review},
  year =         1958,
}

@article{wu2019group,
  title =        {Group Normalization},
  author =       {Yuxin Wu and Kaiming He},
  journal =      {International Journal of Computer Vision},
  year =         2019,
}

@article{cho2014properties,
  author =       {Cho, Kyunghyun and Merrienboer, Bart and Bahdanau, Dzmitry and
                  Bengio, Y.},
  year =         2014,
  title =        {On the Properties of Neural Machine Translation:
                  Encoder-Decoder Approaches},
}

@article{elman1990finding,
  title =        {Finding structure in time},
  journal =      {Cognitive Science},
  year =         1990,
  author =       {Jeffrey L. Elman},
}

@InProceedings{henriques2019small,
  author =       {Henriques, João F. and Ehrhardt, Sebastien and Albanie, Samuel
                  and Vedaldi, Andrea},
  title =        {Small Steps and Giant Leaps: Minimal Newton Solvers for Deep
                  Learning},
  booktitle =    {International Conference on Computer Vision (ICCV)},
  year =         2019
}

@inproceedings{foret2021sharpnessaware,
  title =        {Sharpness-aware Minimization for Efficiently Improving
                  Generalization},
  author =       {Pierre Foret and Ariel Kleiner and Hossein Mobahi and Behnam
                  Neyshabur},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2021,
}

@inproceedings{kim2022fisher,
  author =       {Kim, Minyoung and Li, Da and Hu, Shell and Hospedales,
                  Timothy},
  year =         2022,
  title =        {Fisher SAM: Information Geometry and Sharpness Aware
                  Minimisation},
  booktitle =    {International Conference on Machine Learning (ICML)},
}

@inproceedings{hochreiter1994simplifying,
  author =       {Hochreiter, Sepp and Schmidhuber, J\"{u}rgen},
  booktitle =    {Advances in Neural Information Processing Systems (NIPS)},
  title =        {Simplifying Neural Nets by Discovering Flat Minima},
  year =         1994
}

@inproceedings{jiang2019fantastic,
  author =       {Jiang, Yiding and Neyshabur, Behnam and Mobahi, Hossein and
                  Krishnan, Dilip and Bengio, Samy},
  title =        {Fantastic Generalization Measures and Where to Find Them},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2019,
}

@inproceedings{li2020hessian,
  title =        {Hessian based analysis of {SGD} for deep nets: Dynamics and
                  generalization},
  author =       {Li, Xinyan and Gu, Qilong and Zhou, Yingxue and Chen, Tiancong
                  and Banerjee, Arindam},
  booktitle =    {SIAM International Conference on Data Mining (SMD)},
  year =         2020,
}

@book{colvin2008talent,
  title =        {Talent Is Overrated: What Really Separates World-Class
                  Performers from Everybody Else},
  author =       {Colvin, Geoff},
  year =         2008,
  publisher =    {Penguin Publishing Group}
}

@article{li2017preconditioned,
  title =        {Preconditioned stochastic gradient descent},
  author =       {Li, Xi-Lin},
  journal =      {IEEE Transactions on Neural Networks and Learning Systems},
  year =         2017,
}

@article{adebayo2022towards,
  title =        {Towards Effective Tools for Debugging Machine Learning Models},
  author =       {Adebayo, Julius},
  year =         2022,
}

@inproceedings{petzka2021relative,
  author =       {Petzka, Henning and Kamp, Michael and Adilova, Linara and
                  Sminchisescu, Cristian and Boley, Mario},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  title =        {Relative Flatness and Generalization},
  year =         2021
}

@article{liu2021novel,
  title =        {A Novel Structured Natural Gradient Descent for Deep Learning},
  author =       {Liu, Weihua and Liu, Xiabi},
  year =         2021
}

@article{yang2022sketch,
  title =        {Sketch-Based Empirical Natural Gradient Methods for Deep
                  Learning},
  author =       {Yang, Minghan and Xu, Dong and Wen, Zaiwen and Chen, Mengyun
                  and Xu, Pengxiang},
  journal =      {Journal of Scientific Computing},
  year =         2022,
}

@article{cohen2022adaptive,
  title =        {Adaptive Gradient Methods at the Edge of Stability},
  author =       {Cohen, Jeremy M and Ghorbani, Behrooz and Krishnan, Shankar
                  and Agarwal, Naman and Medapati, Sourabh and Badura, Michal
                  and Suo, Daniel and Cardoze, David and Nado, Zachary and Dahl,
                  George E and others},
  year =         2022
}

@article{cohen2021gradient,
  title =        {Gradient descent on neural networks typically occurs at the
                  edge of stability},
  author =       {Cohen, Jeremy M and Kaur, Simran and Li, Yuanzhi and Kolter, J
                  Zico and Talwalkar, Ameet},
  year =         2021
}

@inproceedings{skorski2021revisiting,
  title =        {Revisiting Weight Initialization of Deep Neural Networks},
  author =       {Skorski, Maciej and Temperoni, Alessandro and Theobald,
                  Martin},
  booktitle =    {Asian Conference on Machine Learning (ACML)},
  year =         2021,
}

@book{griewank2008evaluating,
  title =        {Evaluating derivatives: principles and techniques of
                  algorithmic differentiation},
  author =       {Griewank, Andreas and Walther, Andrea},
  year =         2008,
  publisher =    {SIAM}
}

@article{reiz2022neural,
  title =        {Neural Nets with a Newton Conjugate Gradient Method on
                  Multiple GPUs},
  author =       {Reiz, Severin and Neckel, Tobias and Bungartz, Hans-Joachim},
  year =         2022
}

@article{mohtashami2022avoiding,
  title =        {On Avoiding Local Minima Using Gradient Descent With Large
                  Learning Rates},
  author =       {Mohtashami, Amirkeivan and Jaggi, Martin and Stich, Sebastian},
  year =         2022
}

@article{lorraine2018stochastic,
  title =        {Stochastic hyperparameter optimization through hypernetworks},
  author =       {Lorraine, Jonathan and Duvenaud, David},
  year =         2018
}

@article{karakida2020understanding,
  title =        {Understanding approximate fisher information for fast
                  convergence of natural gradient descent in wide neural
                  networks},
  author =       {Karakida, Ryo and Osawa, Kazuki},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2020
}

@article{raab2022conjugate,
  title =        {Conjugate Natural Selection},
  author =       {Raab, Reilly and de Alfaro, Luca and Liu, Yang},
  year =         2022
}

@inproceedings{ravishankar2022stochastic,
  title =        {Stochastic Weight Perturbations Along the Hessian: A
                  Plug-and-Play Method to Compute Uncertainty},
  author =       {Ravishankar, Hariharan and Patil, Rohan and Anand, Deepa and
                  Singhal, Vanika and Agrawal, Utkarsh and Venkataramani, Rahul
                  and Sudhakar, Prasad},
  booktitle =    {International Workshop on Uncertainty for Safe Utilization of
                  Machine Learning in Medical Imaging},
  year =         2022,
}

@inproceedings{zhu2022hessian,
  title =        {Heslsian-Aided Random Perturbation (HARP) Using Noisy
                  Zeroth-Order Queries},
  author =       {Zhu, Jingyi},
  booktitle =    {Mathematical and Scientific Machine Learning (MSML)},
  year =         2022,
}

@article{chaskalovic2022refined,
  title =        {A refined first-order expansion formula in Rn: Application to
                  interpolation and finite element error estimates},
  author =       {Chaskalovic, Joel and Assous, Franck},
  year =         2022
}

@article{fawzi2022discovering,
  title =        {Discovering faster matrix multiplication algorithms with
                  reinforcement learning},
  author =       {Fawzi, Alhussein and Balog, Matej and Huang, Aja and Hubert,
                  Thomas and Romera-Paredes, Bernardino and Barekatain,
                  Mohammadamin and Novikov, Alexander and R Ruiz, Francisco J
                  and Schrittwieser, Julian and Swirszcz, Grzegorz and others},
  journal =      {Nature},
  year =         2022,
}

@book{kommer2018souveraen,
  title =        {Souver{\"a}n investieren f{\"u}r Einsteiger: Wie Sie mit ETFs
                  ein Verm{\"o}gen bilden},
  author =       {Kommer, Gerd},
  year =         2018,
}

@inproceedings{chen2022efficient,
  title =        {Efficient Second-Order Optimization for Neural Networks with
                  Kernel Machines},
  author =       {Chen, Yawen and Chen, Yile and Chen, Jian and Wen, Zeyi and
                  Huang, Jin},
  booktitle =    {ACM International Conference on Information \& Knowledge
                  Management},
  year =         2022
}

@article{pleiss2020fast,
  title =        {Fast matrix square roots with applications to Gaussian
                  processes and Bayesian optimization},
  author =       {Pleiss, Geoff and Jankowiak, Martin and Eriksson, David and
                  Damle, Anil and Gardner, Jacob},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2020
}

@article{advani2020high-dimensional,
  title =        {High-dimensional dynamics of generalization error in neural
                  networks},
  journal =      {Neural Networks},
  year =         2020,
  author =       {Madhu S. Advani and Andrew M. Saxe and Haim Sompolinsky},
}

@inproceedings{martens2012estimating,
  author =       {Martens, James and Sutskever, Ilya and Swersky, Kevin},
  title =        {Estimating the Hessian by Back-Propagating Curvature},
  year =         2012,
  booktitle =    {International Conference on Machine Learning (ICML)},
}

@book{breymann2015cpp,
  title =        {Der C++-Programmierer: C++ lernen--professionell
                  anwenden--L{\"o}sungen nutzen},
  author =       {Breymann, Ulrich},
  year =         2015,
  publisher =    {Carl Hanser Verlag GmbH Co KG}
}

@book{sanders2010cuda,
  title =        {CUDA by example: an introduction to general-purpose GPU
                  programming},
  author =       {Sanders, Jason and Kandrot, Edward},
  year =         2010,
  publisher =    {Addison-Wesley Professional}
}

@article{dumoulin2016guide,
  title =        {A guide to convolution arithmetic for deep learning},
  author =       {Dumoulin, Vincent and Visin, Francesco},
  year =         2016
}

@article{roosta2019sub,
  title =        {Sub-sampled Newton methods},
  author =       {Roosta-Khorasani, Farbod and Mahoney, Michael W},
  journal =      {Mathematical Programming},
  year =         2019,
}

@inproceedings{laue2020simple,
  title =        {A simple and efficient tensor calculus},
  author =       {Laue, S{\"o}ren and Mitterreiter, Matthias and Giesen,
                  Joachim},
  booktitle =    {AAAI Conference on Artificial Intelligence},
  year =         2020
}

@article{ruckstiess2011python,
  title =        {A Python Experiment Suite},
  author =       {R{\"u}ckstie{\ss}, Thomas and Schmidhuber, J{\"u}rgen},
  journal =      {The Python Papers},
  year =         2011
}

@inproceedings{elsayed2023hesscale,
  title =        {{HesScale}: Scalable Computation of Hessian Diagonals},
  author =       {Mohamed Elsayed and A. Rupam Mahmood},
  year =         2023,
}

@inproceedings{klaus2022convexity,
  title =        {Convexity Certificates from Hessians},
  author =       {Julien Klaus and Niklas Merk and Konstantin Wiedom and
                  S{\"o}ren Laue and Joachim Giesen},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2022,
}

@inproceedings{antonakopoulos2022extranewton,
  title =        {Extra-Newton: A First Approach to Noise-Adaptive Accelerated
                  Second-Order Methods},
  author =       {Kimon Antonakopoulos and Ali Kavis and Volkan Cevher},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2022,
}

@book{cormen2009introduction,
  title =        {Introduction to algorithms},
  author =       {Cormen, Thomas H and Leiserson, Charles E and Rivest, Ronald L
                  and Stein, Clifford},
  year =         2009,
  publisher =    {MIT press}
}

@inproceedings{fang2022an,
  title =        {An In-depth Study of Stochastic Backpropagation},
  author =       {Jun Fang and Mingze Xu and Hao Chen and Bing Shuai and Zhuowen
                  Tu and Joseph Tighe},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2022,
}

@inproceedings{srinivas2022efficient,
  title =        {Efficient Training of Low-Curvature Neural Networks},
  author =       {Suraj Srinivas and Kyle Matoba and Himabindu Lakkaraju and
                  Fran{\c{c}}ois Fleuret},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2022,
}

@inproceedings{chandra2022gradient,
  title =        {Gradient Descent: The Ultimate Optimizer},
  author =       {Kartik Chandra and Audrey Xie and Jonathan Ragan-Kelley and
                  Erik Meijer},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2022,
}

@article{laue2018computing,
  title =        {Computing higher order derivatives of matrix and tensor
                  expressions},
  author =       {Laue, S{\"o}ren and Mitterreiter, Matthias and Giesen,
                  Joachim},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2018
}

@inproceedings{ma2020autohoot,
  title =        {Autohoot: Automatic high-order optimization for tensors},
  author =       {Ma, Linjian and Ye, Jiayu and Solomonik, Edgar},
  booktitle =    {International Conference on Parallel Architectures and
                  Compilation Techniques (PACT)},
  year =         2020
}

@inproceedings{alwani2016fused,
  title =        {Fused-layer CNN accelerators},
  author =       {Alwani, Manoj and Chen, Han and Ferdman, Michael and Milder,
                  Peter},
  booktitle =    {International Symposium on Microarchitecture (MICRO)},
  year =         2016,
}

@inproceedings{rogozhnikov2022einops,
  title =        {Einops: Clear and Reliable Tensor Manipulations with
                  Einstein-like Notation},
  author =       {Alex Rogozhnikov},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2022,
}

@article{solomonik2014massively,
  title =        {A massively parallel tensor contraction framework for
                  coupled-cluster computations},
  author =       {Solomonik, Edgar and Matthews, Devin and Hammond, Jeff R and
                  Stanton, John F and Demmel, James},
  journal =      {Journal of Parallel and Distributed Computing},
  year =         2014,
}

@article{tran2022gradient,
  title =        {Gradient Descent-Type Methods--Background and Simple Unified
                  Convergence Analysis},
  author =       {Tran-Dinh, Quoc and van Dijk, Marten},
  year =         2022,
}

@article{aehle2022reverse,
  title =        {Reverse-Mode Automatic Differentiation of Compiled Programs},
  author =       {Aehle, Max and Bl{\"u}hdorn, Johannes and Sagebaum, Max and
                  Gauger, Nicolas R},
  year =         2022
}

@book{walker2017we,
  title =        {Why we sleep: Unlocking the power of sleep and dreams},
  author =       {Walker, Matthew},
  year =         2017,
}

@article{dangel2022vivit,
  title =        {Vi{V}i{T}: Curvature Access Through The Generalized
                  Gauss-Newton{\textquoteright}s Low-Rank Structure},
  author =       {Felix Dangel and Lukas Tatzel and Philipp Hennig},
  journal =      {Transactions on Machine Learning Research (TMLR)},
  year =         2022,
  keywords =     {mywork},
}

@article{betancourt2018geometric,
  title =        {A geometric theory of higher-order automatic differentiation},
  author =       {Betancourt, Michael},
  year =         2018
}

@article{sclocchi2023dissecting,
  title =        {Dissecting the Effects of SGD Noise in Distinct Regimes of
                  Deep Learning},
  author =       {Sclocchi, Antonio and Geiger, Mario and Wyart, Matthieu},
  year =         2023
}

@inproceedings{obermeyer2019functional,
  title =        {Functional Tensors for Probabilistic Programming},
  author =       {Fritz Obermeyer and Eli Bingham and Martin Jankowiak and Du
                  Phan and Jonathan Chen},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS),
                  Workshop Program Transformations},
  year =         2019,
}

@article{arbel2023rethinking,
  title =        {Rethinking Gauss-Newton for learning over-parameterized
                  models},
  author =       {Arbel, Michael},
  year =         2023
}

@article{baumgartner2023local,
  title =        {Local Convergence Behaviour of Generalized Gauss-Newton
                  Multiple Shooting, Single Shooting and Differential Dynamic
                  Programming},
  author =       {Baumg{\"a}rtner, Katrin and Messerer, Florian and Diehl,
                  Moritz},
  year =         2023
}

@inproceedings{tatzel2022late,
  title =        {Late-Phase Second-Order Training},
  author =       {Tatzel, Lukas and Hennig, Philipp and Schneider, Frank},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS),
                  Workshop Has it Trained Yet?},
  year =         2022
}

@misc{lu2022indiscriminate,
  title =        {Indiscriminate Data Poisoning Attacks on Neural Networks},
  author =       {Lu, Yiwei and Kamath, Gautam and Yu, Yaoliang},
  year =         2022
}

@misc{lu2023exploring,
  author =       {Yiwei Lu, Gautam Kamath, Yaoliang Yu},
  title =        {Exploring the Limits of Indiscriminate Data Poisoning Attacks},
  year =         2023,
}

@inproceedings{petersen2023isaac,
  title =        {{ISAAC} Newton: Input-based Approximate Curvature for Newton's
                  Method},
  author =       {Felix Petersen and Tobias Sutter and Christian Borgelt and
                  Dongsung Huh and Hilde Kuehne and Yuekai Sun and Oliver
                  Deussen},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2023,
}

@inproceedings{peirson2022fishy,
  title =        {Fishy: Layerwise Fisher Approximation for Higher-order Neural
                  Network Optimization},
  author =       {Peirson, Abel and Amid, Ehsan and Chen, Yatong and Feinberg,
                  Vladimir and Warmuth, Manfred K and Anil, Rohan},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS),
                  Workshop Has it Trained Yet?},
  year =         2022,
}

@inproceedings{krizhevsky2012imagenet,
  author =       {Krizhevsky, Alex and Sutskever, Ilya and Hinton, Geoffrey E.},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  title =        {ImageNet Classification with Deep Convolutional Neural
                  Networks},
  year =         2012
}

@article{simonyan2014very,
  title =        {Very deep convolutional networks for large-scale image
                  recognition},
  author =       {Simonyan, Karen and Zisserman, Andrew},
  year =         2014
}

@inproceedings{he2016deep,
  title =        {Deep residual learning for image recognition},
  author =       {He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun,
                  Jian},
  booktitle =    {IEEE conference on computer vision and pattern recognition
                  (CVPR)},
  year =         2016
}

@misc{sivan2023fosi,
  author =       {Sivan, Hadar and Gabel, Moshe and Schuster, Assaf},
  title =        {FOSI: Hybrid First and Second Order Optimization},
  year =         2023,
}

@inproceedings{fradkin2022robustness,
  title =        {Robustness to Adversarial Gradients: A Glimpse Into the Loss
                  Landscape of Contrastive Pre-training},
  author =       {Philip Fradkin and Lazar Atanackovic and Michael R. Zhang},
  booktitle =    {International Conference on Machine Learning (ICML), Workshop
                  Pre-training: Perspectives, Pitfalls, and Paths Forward},
  year =         2022,
}

@inproceedings{kunin2021neural,
  title =        {Neural Mechanics: Symmetry and Broken Conservation Laws in
                  Deep Learning Dynamics},
  author =       {Daniel Kunin and Javier Sagastuy-Brena and Surya Ganguli and
                  Daniel LK Yamins and Hidenori Tanaka},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2021,
}

@inproceedings{zhao2022symmetry,
  title =        {Symmetry Teleportation for Accelerated Optimization},
  author =       {Bo Zhao and Nima Dehmamy and Robin Walters and Rose Yu},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2022,
}

@inproceedings{zhao2023symmetries,
  title =        {Symmetries, Flat Minima and the Conserved Quantities of
                  Gradient Flow},
  author =       {Bo Zhao and Iordan Ganev and Robin Walters and Rose Yu and
                  Nima Dehmamy},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2023,
}

@inproceedings{chellapilla2006high,
  title =        {High performance convolutional neural networks for document
                  processing},
  author =       {Chellapilla, Kumar and Puri, Sidd and Simard, Patrice},
  booktitle =    {International Workshop on Frontiers in Handwriting
                  Recognition},
  year =         2006,
}

@article{lenton2021ivy,
  title =        {Ivy: Templated deep learning for inter-framework portability},
  author =       {Lenton, Daniel and Pardo, Fabio and Falck, Fabian and James,
                  Stephen and Clark, Ronald},
  year =         2021
}

@book{sussman2013functional,
  title =        {Functional differential geometry},
  author =       {Sussman, Gerald Jay and Wisdom, Jack},
  year =         2013,
  publisher =    {MIT Press}
}

@article{singh2021analytic,
  title =        {Analytic insights into structure and rank of neural network
                  hessian maps},
  author =       {Singh, Sidak Pal and Bachmann, Gregor and Hofmann, Thomas},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2021
}

@inproceedings{frantar2022optimal,
  title =        {Optimal Brain Compression: A Framework for Accurate
                  Post-Training Quantization and Pruning},
  author =       {Elias Frantar and Dan Alistarh},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2022,
}

@inproceedings{ainsworth2023git,
  title =        {Git Re-Basin: Merging Models modulo Permutation Symmetries},
  author =       {Samuel Ainsworth and Jonathan Hayase and Siddhartha Srinivasa},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2023,
}

@misc{power2022grokking,
  author =       {Power, Alethea and Burda, Yuri and Edwards, Harri and
                  Babuschkin, Igor and Misra, Vedant},
  title =        {Grokking: Generalization Beyond Overfitting on Small
                  Algorithmic Datasets},
  year =         2022,
}

@inproceedings{han2022neural,
  title =        {Neural Collapse Under {MSE} Loss: Proximity to and Dynamics on
                  the Central Path},
  author =       {X.Y. Han and Vardan Papyan and David L. Donoho},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2022,
}

@inproceedings{hataya2023nystrom,
  author =       {Hataya, Ryuichiro and Yamada, Makoto},
  title =        {Nystrom Method for Accurate and Scalable Implicit
                  Differentiation},
  booktitle =    {International Conference on Artificial Intelligence and
                  Statistics (AISTATS)},
  year =         2023,
}

@article{pineda2022theseus,
  title =        {{Theseus: A Library for Differentiable Nonlinear
                  Optimization}},
  author =       {Luis Pineda and Taosha Fan and Maurizio Monge and Shobha
                  Venkataraman and Paloma Sodhi and Ricky TQ Chen and Joseph
                  Ortiz and Daniel DeTone and Austin Wang and Stuart Anderson
                  and Jing Dong and Brandon Amos and Mustafa Mukadam},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2022
}

@inproceedings{tishby2015deep,
  author =       {Tishby, Naftali and Zaslavsky, Noga},
  booktitle =    {IEEE Information Theory Workshop (ITW)},
  title =        {Deep learning and the information bottleneck principle},
  year =         2015,
}

@article{dangel2023backpropagation,
  title =        {Backpropagation Beyond the Gradient},
  author =       {Dangel, Felix Julius},
  year =         2023,
  school =       {Universit{\"a}t T{\"u}bingen},
  keywords =     {mywork},
}

@article{ward2023improving,
  title =        {Improving the Performance and Stability of TIC and ICE},
  author =       {Ward, Tyler},
  journal =      {Entropy},
  year =         2023,
}

@article{orvieto2021vanishing,
  title =        {Vanishing curvature and the power of adaptive methods in
                  randomly initialized deep networks},
  author =       {Orvieto, Antonio and Kohler, Jonas and Pavllo, Dario and
                  Hofmann, Thomas and Lucchi, Aurelien},
  year =         2021
}

@inproceedings{kunstner2023noise,
  title =        {Noise Is Not the Main Factor Behind the Gap Between Sgd and
                  Adam on Transformers, But Sign Descent Might Be},
  author =       {Frederik Kunstner and Jacques Chen and Jonathan Wilder
                  Lavington and Mark Schmidt},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2023,
}

@article{godfrey2022symmetries,
  title =        {On the symmetries of deep learning models and their internal
                  representations},
  author =       {Godfrey, Charles and Brown, Davis and Emerson, Tegan and
                  Kvinge, Henry},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2022
}

@inproceedings{simsek2021geometry,
  title =        {Geometry of the loss landscape in overparameterized neural
                  networks: Symmetries and invariances},
  author =       {Simsek, Berfin and Ged, Fran{\c{c}}ois and Jacot, Arthur and
                  Spadaro, Francesco and Hongler, Cl{\'e}ment and Gerstner,
                  Wulfram and Brea, Johanni},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2021,
}

@misc{laue2022equivalence,
  title =        {On the Equivalence of Automatic and Symbolic Differentiation},
  author =       {Soeren Laue},
  year =         2022,
}

@misc{koroko2023analysis,
  title =        {Analysis and Comparison of Two-Level KFAC Methods for Training
                  Deep Neural Networks},
  author =       {Abdoulaye Koroko and Ani Anciaux-Sedrakian and Ibtihel Ben
                  Gharbia and Valérie Garès and Mounir Haddou and Quang Huy
                  Tran},
  year =         2023,
}

@article{gray2021hyper,
  title =        {Hyper-optimized tensor network contraction},
  author =       {Gray, Johnnie and Kourtis, Stefanos},
  journal =      {Quantum},
  year =         2021,
}

@inproceedings{song2022fast,
  title =        {Fast Differentiable Matrix Square Root},
  author =       {Yue Song and Nicu Sebe and Wei Wang},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2022,
}

@article{song2022fast2,
  title =        {Fast Differentiable Matrix Square Root and Inverse Square
                  Root},
  author =       {Song, Yue and Sebe, Nicu and Wang, Wei},
  journal =      {IEEE Transactions on Pattern Analysis and Machine Intelligence
                  (TPAMI)},
  year =         2022,
}

@article{cheng2005minimization,
  author =       {Hsiao-Bing Cheng and Li-Tien Cheng and Shing-Tung Yau},
  title =        {{Minimization with the affine normal direction}},
  journal =      {Communications in Mathematical Sciences},
  year =         2005,
}

@article{nilsen2019efficient,
  title =        {Efficient computation of hessian matrices in tensorflow},
  author =       {Nilsen, Geir K and Munthe-Kaas, Antonella Z and Skaug, Hans J
                  and Brun, Morten},
  year =         2019
}

@article{baydin2022gradients,
  title =        {Gradients without backpropagation},
  author =       {Baydin, At{\i}l{\i}m G{\"u}ne{\c{s}} and Pearlmutter, Barak A
                  and Syme, Don and Wood, Frank and Torr, Philip},
  year =         2022
}

@article{streeter2022autobound,
  title =        {Automatically Bounding the Taylor Remainder Series: Tighter
                  Bounds and New Applications},
  author =       {Streeter, Matthew and Dillon, Joshua V},
  year =         2022
}

@misc{page2020learning,
  author =       {Page, Josue and Saltarin, Federico and Belyaev, Yury and Lyck,
                  Ruth and Favaro, Paolo},
  title =        {Learning to Reconstruct Confocal Microscope Stacks from Single
                  Light Field Images},
  year =         2020,
}

@inproceedings{zhang2020convolutional,
  title =        {V4D: 4D Convolutional Neural Networks for Video-level
                  Representation Learning},
  author =       {Shiwen Zhang and Sheng Guo and Weilin Huang and Matthew R.
                  Scott and Limin Wang},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2020,
}

@article{bengs2020spatio-temporal,
  title =        {4D spatio-temporal convolutional networks for object position
                  estimation in OCT volumes},
  author =       {Marcel Bengs and Nils Gessert and Alexander Schlaefer},
  journal =      {Current Directions in Biomedical Engineering},
  year =         2020,
}

@inproceedings{li2022omni-dimensional,
  title =        {Omni-Dimensional Dynamic Convolution},
  author =       {Chao Li and Aojun Zhou and Anbang Yao},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2022,
}

@article{huang2021efficient,
  title =        {Efficient parallelization of tensor network contraction for
                  simulating quantum computation},
  author =       {Huang, Cupjin and Zhang, Fang and Newman, Michael and Ni,
                  Xiaotong and Ding, Dawei and Cai, Junjie and Gao, Xun and
                  Wang, Tenghui and Wu, Feng and Zhang, Gengyan and others},
  journal =      {Nature Computational Science},
  year =         2021,
}

@article{zhang2020parallel,
  title =        {A Parallel Tensor Network Contraction Algorithm and Its
                  Applications in Quantum Computation},
  author =       {Zhang, Fang},
  year =         2020
}

@article{chen2018classical,
  title =        {Classical simulation of intermediate-size quantum circuits},
  author =       {Chen, Jianxin and Zhang, Fang and Huang, Cupjin and Newman,
                  Michael and Shi, Yaoyun},
  year =         2018
}

@software{nvidia2023cuquantum,
  author       = {The cuQuantum development team},
  title        = {cu{Q}uantum {SDK}: A High-Performance Library for Accelerating
                  Quantum Information Science},
  year         = 2023,
}

@misc{fu2023learning,
  title =        {Learning Trajectories are Generalization Indicators},
  author =       {Jingwen Fu and Zhizheng Zhang and Dacheng Yin and Yan Lu and
                  Nanning Zheng},
  year =         2023,
}

@article{penrose1971applications,
  title =        {Applications of negative dimensional tensors},
  author =       {Penrose, Roger},
  journal =      {Combinatorial Mathematics and its Applications},
  year =         1971
}

@article{lin2023simplifying,
  title =        {Simplifying Momentum-based Riemannian Submanifold
                  Optimization},
  author =       {Lin, Wu and Duruisseaux, Valentin and Leok, Melvin and
                  Nielsen, Frank and Khan, Mohammad Emtiyaz and Schmidt, Mark},
  year =         2023
}

@article{bridgeman2017hand,
  title =        {Hand-waving and interpretive dance: an introductory course on
                  tensor networks},
  author =       {Bridgeman, Jacob C and Chubb, Christopher T},
  journal =      {Journal of Physics A: Mathematical and theoretical},
  year =         2017,
}

@inproceedings{adelman2021faster,
  author =       {Adelman, Menachem and Levy, Kfir and Hakimi, Ido and
                  Silberstein, Mark},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  title =        {Faster Neural Network Training with Approximate Tensor
                  Operations},
  year =         2021
}

@misc{huchette2023deep,
  title =        {When Deep Learning Meets Polyhedral Theory: A Survey},
  author =       {Joey Huchette and Gonzalo Muñoz and Thiago Serra and Calvin
                  Tsay},
  year =         2023,
}

@article{cho2017riemannian,
  title =        {Riemannian approach to batch normalization},
  author =       {Cho, Minhyung and Lee, Jaehyung},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2017
}

@misc{osawa2023asdl,
  title =        {ASDL: A Unified Interface for Gradient Preconditioning in
                  PyTorch},
  author =       {Kazuki Osawa and Satoki Ishikawa and Rio Yokota and Shigang Li
                  and Torsten Hoefler},
  year =         2023,
}

@article{snider2023operator,
  title =        {Operator Fusion in XLA: Analysis and Evaluation},
  author =       {Snider, Daniel and Liang, Ruofan},
  year =         2023
}

@article{edelman1998geometry,
  title =        {The geometry of algorithms with orthogonality constraints},
  author =       {Edelman, Alan and Arias, Tom{\'a}s A and Smith, Steven T},
  journal =      {SIAM journal on Matrix Analysis and Applications (SIMAX)},
  year =         1998,
}

@article{singh2023hessian,
  title =        {The Hessian perspective into the Nature of Convolutional
                  Neural Networks},
  author =       {Sidak Pal Singh and Thomas Hofmann and Bernhard Schölkopf},
  year =         2023,
  booktitle =    {International Conference on Machine Learning (ICML)},
}

@inproceedings{szegedy2016rethinking,
  title =        {Rethinking the inception architecture for computer vision},
  author =       {Szegedy, Christian and Vanhoucke, Vincent and Ioffe, Sergey
                  and Shlens, Jon and Wojna, Zbigniew},
  booktitle =    {IEEE conference on computer vision and pattern recognition
                  (CVPR)},
  year =         2016
}

@inproceedings{wang2020orthogonal,
  title =        {Orthogonal convolutional neural networks},
  author =       {Wang, Jiayun and Chen, Yubei and Chakraborty, Rudrasis and Yu,
                  Stella X},
  booktitle =    {IEEE/CVF conference on computer vision and pattern recognition
                  (CVPR)},
  year =         2020
}

@article{ren2021tensor,
  title =        {Tensor normal training for deep learning models},
  author =       {Ren, Yi and Goldfarb, Donald},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2021
}

@inproceedings{onken2021ot,
  title =        {Ot-flow: Fast and accurate continuous normalizing flows via
                  optimal transport},
  author =       {Onken, Derek and Fung, Samy Wu and Li, Xingjian and Ruthotto,
                  Lars},
  booktitle =    {AAAI Conference on Artificial Intelligence},
  year =         2021
}

@misc{sabne2020xla,
  title =        {XLA : Compiling Machine Learning for Peak Performance},
  author =       {Amit Sabne},
  year =         2020
}

@InProceedings{lee2023exact,
  title =        {Exact Gradient Computation for Spiking Neural Networks via
                  Forward Propagation},
  author =       {Lee, Jane H. and Haghighatshoar, Saeid and Karbasi, Amin},
  booktitle =    {International Conference on Artificial Intelligence and
                  Statistics (AISTATS)},
  year =         2023,
}

@InProceedings{xu2023linear,
  title =        {Linear Convergence of Gradient Descent For Finite Width
                  Over-parametrized Linear Networks With General Initialization},
  author =       {Xu, Ziqing and Min, Hancheng and Tarmoun, Salma and Mallada,
                  Enrique and Vidal, Rene},
  booktitle =    {International Conference on Artificial Intelligence and
                  Statistics (AISTATS)},
  year =         2023,
}

@InProceedings{kiral2023lie,
  title =        {The Lie-Group Bayesian Learning Rule},
  author =       {Kiral, Eren Mehmet and Moellenhoff, Thomas and Khan, Mohammad
                  Emtiyaz},
  booktitle =    {International Conference on Artificial Intelligence and
                  Statistics (AISTATS)},
  year =         2023,
}

@InProceedings{meller2023singular,
  title =        {Singular Value Representation: A New Graph Perspective On
                  Neural Networks},
  author =       {Meller, Dan and Berkouk, Nicolas},
  booktitle =    {International Conference on Artificial Intelligence and
                  Statistics (AISTATS)},
  year =         2023,
}

@InProceedings{karjol2023neural,
  title =        {Neural Discovery of Permutation Subgroups},
  author =       {Karjol, Pavan and Kashyap, Rohan and {AP}, Prathosh},
  booktitle =    {International Conference on Artificial Intelligence and
                  Statistics (AISTATS)},
  year =         2023,
}

@InProceedings{wang2023loft,
  title =        {LOFT: Finding Lottery Tickets through Filter-wise Training},
  author =       {Wang, Qihan and Dun, Chen and Liao, Fangshuo and Jermaine,
                  Chris and Kyrillidis, Anastasios},
  booktitle =    {International Conference on Artificial Intelligence and
                  Statistics (AISTATS)},
  year =         2023,
}

@InProceedings{xu2023accelerated,
  title =        {On the Accelerated Noise-Tolerant Power Method},
  author =       {Xu, Zhiqiang},
  booktitle =    {International Conference on Artificial Intelligence and
                  Statistics (AISTATS)},
  year =         2023,
}

@InProceedings{bahamou2023mini,
  title =        {A Mini-Block Fisher Method for Deep Neural Networks},
  author =       {Bahamou, Achraf and Goldfarb, Donald and Ren, Yi},
  booktitle =    {International Conference on Artificial Intelligence and
                  Statistics (AISTATS)},
  year =         2023,
}

@InProceedings{khramtsova2023convolutional,
  title =        {Convolutional Persistence as a Remedy to Neural Model
                  Analysis},
  author =       {Khramtsova, Ekaterina and Zuccon, Guido and Wang, Xi and
                  Baktashmotlagh, Mahsa},
  booktitle =    {International Conference on Artificial Intelligence and
                  Statistics (AISTATS)},
  year =         2023,
}

@misc{balboni2023adler,
  title =        {{ADLER} -- An efficient Hessian-based strategy for adaptive
                  learning rate},
  author =       {Dario Balboni and Davide Bacciu},
  year =         2023,
}

@inproceedings{wang2019satnet,
  title =        {Satnet: Bridging deep learning and logical reasoning using a
                  differentiable satisfiability solver},
  author =       {Wang, Po-Wei and Donti, Priya and Wilder, Bryan and Kolter,
                  Zico},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2019,
}

@inproceedings{novak2022fast,
  title =        {Fast Finite Width Neural Tangent Kernel},
  author =       {Roman Novak and Jascha Sohl-Dickstein and Samuel S.
                  Schoenholz},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2022,
}

@inproceedings{garcia2023fisherlegendre,
  title =        {Fisher-Legendre (FishLeg) optimization of deep neural
                  networks},
  author =       {Jezabel R Garcia and Federica Freddi and Stathi Fotiadis and
                  Maolin Li and Sattar Vakili and Alberto Bernacchia and
                  Guillaume Hennequin},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2023,
}

@inproceedings{li2023what,
  title =        {What Makes Convolutional Models Great on Long Sequence
                  Modeling?},
  author =       {Yuhong Li and Tianle Cai and Yi Zhang and Deming Chen and
                  Debadeepta Dey},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2023,
}

@inproceedings{trockman2023understanding,
  title =        {Understanding the Covariance Structure of Convolutional
                  Filters},
  author =       {Asher Trockman and Devin Willmott and J Zico Kolter},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2023,
}

@inproceedings{murray2023characterizing,
  title =        {Characterizing the spectrum of the {NTK} via a power series
                  expansion},
  author =       {Michael Murray and Hui Jin and Benjamin Bowman and Guido
                  Montufar},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2023,
}

@inproceedings{liu2023more,
  title =        {More ConvNets in the 2020s: Scaling up Kernels Beyond 51x51
                  using Sparsity},
  author =       {Shiwei Liu and Tianlong Chen and Xiaohan Chen and Xuxi Chen
                  and Qiao Xiao and Boqian Wu and Tommi K{\"a}rkk{\"a}inen and
                  Mykola Pechenizkiy and Decebal Constantin Mocanu and Zhangyang
                  Wang},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2023,
}

@inproceedings{chen2023which,
  title =        {Which Layer is Learning Faster? A Systematic Exploration of
                  Layer-wise Convergence Rate for Deep Neural Networks},
  author =       {Yixiong Chen and Alan Yuille and Zongwei Zhou},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2023,
}

@inproceedings{chmiel2023accurate,
  title =        {Accurate Neural Training with 4-bit Matrix Multiplications at
                  Standard Formats},
  author =       {Brian Chmiel and Ron Banner and Elad Hoffer and Hilla
                  Ben-Yaacov and Daniel Soudry},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2023,
}

@inproceedings{zhang2023eva,
  title =        {Eva: Practical Second-order Optimization with
                  Kronecker-vectorized Approximation},
  author =       {Lin Zhang and Shaohuai Shi and Bo Li},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2023,
}

@inproceedings{chen2023dropit,
  title =        {Drop{IT}: Dropping Intermediate Tensors for Memory-Efficient
                  {DNN} Training},
  author =       {Joya Chen and Kai Xu and Yuhui Wang and Yifei Cheng and Angela
                  Yao},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2023,
}

@inproceedings{choe2023betty,
  title =        {Betty: An Automatic Differentiation Library for Multilevel
                  Optimization},
  author =       {Sang Keun Choe and Willie Neiswanger and Pengtao Xie and Eric
                  Xing},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2023,
}

@inproceedings{zhang2018stabilizing,
  title =        {Stabilizing gradients for deep neural networks via efficient
                  svd parameterization},
  author =       {Zhang, Jiong and Lei, Qi and Dhillon, Inderjit},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2018,
}

@article{vander2021parameter,
  title =        {Parameter efficient neural networks with singular value
                  decomposed kernels},
  author =       {Vander Mijnsbrugge, David and Ongenae, Femke and Van Hoecke,
                  Sofie},
  journal =      {IEEE Transactions on Neural Networks and Learning Systems},
  year =         2021,
}

@inproceedings{dyro2022second,
  title =        {Second-Order Sensitivity Analysis for Bilevel Optimization},
  author =       {Dyro, Robert and Schmerling, Edward and Arechiga, Nikos and
                  Pavone, Marco},
  booktitle =    {International Conference on Artificial Intelligence and
                  Statistics (AISTATS)},
  year =         2022,
}

@article{mei2023kradagrad,
  title =        {KrADagrad: Kronecker Approximation-Domination Gradient
                  Preconditioned Stochastic Optimization},
  author =       {Mei, Jonathan and Moreno, Alexander and Walters, Luke},
  booktitle =    {Conference on Uncertainty in Artificial Intelligence (UAI)},
  year =         2023
}

@article{papyan2016convolutional,
  title =        {Convolutional neural networks analyzed via convolutional
                  sparse coding},
  author =       {Papyan, Vardan and Romano, Yaniv and Elad, Michael},
  year =         2016
}

@software{dangel2021unfoldnd,
  author =       {Felix Dangel},
  title =        {unfold{N}d: (N=1,2,3)-dimensional unfold (im2col) and fold
                  (col2im) in PyTorch},
  year =         2021,
  journal =      {GitHub repository (https://github.com/f-dangel/unfoldNd)},
  keywords =     {mywork},
}

@inproceedings{kjolstad2017taco,
  title =        {Taco: A tool to generate tensor algebra kernels},
  author =       {Kjolstad, Fredrik and Chou, Stephen and Lugato, David and
                  Kamil, Shoaib and Amarasinghe, Saman},
  booktitle =    {IEEE/ACM International Conference on Automated Software
                  Engineering (ASE)},
  year =         2017,
}

@article{liu2022convnet,
  author =       {Zhuang Liu and Hanzi Mao and Chao-Yuan Wu and Christoph
                  Feichtenhofer and Trevor Darrell and Saining Xie},
  title =        {A ConvNet for the 2020s},
  journal =      {IEEE/CVF Conference on Computer Vision and Pattern Recognition
                  (CVPR)},
  year =         2022,
}

@article{yang2019condconv,
  title =        {Condconv: Conditionally parameterized convolutions for
                  efficient inference},
  author =       {Yang, Brandon and Bender, Gabriel and Le, Quoc V and Ngiam,
                  Jiquan},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2019
}

@inproceedings{chen2020dynamic,
  title =        {Dynamic convolution: Attention over convolution kernels},
  author =       {Chen, Yinpeng and Dai, Xiyang and Liu, Mengchen and Chen,
                  Dongdong and Yuan, Lu and Liu, Zicheng},
  booktitle =    {IEEE/CVF conference on computer vision and pattern recognition
                  (CVPR)},
  year =         2020
}

@inproceedings{hu2018squeeze,
  title =        {Squeeze-and-excitation networks},
  author =       {Hu, Jie and Shen, Li and Sun, Gang},
  booktitle =    {IEEE/CVF conference on computer vision and pattern recognition
                  (CVPR)},
  year =         2018
}

@article{smith1997scientist,
  title =        {The Scientist and Engineer's Guide to Digital Signal
                  Processing},
  author =       {Smith, Steven W.},
  year =         1997
}

@inproceedings{rigamonti2013learning,
  author =       {Rigamonti, Roberto and Sironi, Amos and Lepetit, Vincent and
                  Fua, Pascal},
  booktitle =    {IEEE Conference on Computer Vision and Pattern Recognition
                  (CVPR)},
  title =        {Learning Separable Filters},
  year =         2013,
}

@article{tai2015convolutional,
  title =        {Convolutional neural networks with low-rank regularization},
  author =       {Tai, Cheng and Xiao, Tong and Zhang, Yi and Wang, Xiaogang and
                  others},
  year =         2015
}

@article{howard2017mobilenets,
  title =        {Mobilenets: Efficient convolutional neural networks for mobile
                  vision applications},
  author =       {Howard, Andrew G and Zhu, Menglong and Chen, Bo and
                  Kalenichenko, Dmitry and Wang, Weijun and Weyand, Tobias and
                  Andreetto, Marco and Adam, Hartwig},
  year =         2017
}

@article{sifre2014rigid,
  title =        {Rigid-motion scattering for texture classification},
  author =       {Sifre, Laurent and Mallat, St{\'e}phane},
  year =         2014
}

@inproceedings{chollet2017xception,
  title =        {Xception: Deep learning with depthwise separable convolutions},
  author =       {Chollet, Fran{\c{c}}ois},
  booktitle =    {IEEE conference on computer vision and pattern recognition
                  (CVPR)},
  year =         2017
}

@inproceedings{sandler2018mobilenetv2,
  title =        {Mobilenetv2: Inverted residuals and linear bottlenecks},
  author =       {Sandler, Mark and Howard, Andrew and Zhu, Menglong and
                  Zhmoginov, Andrey and Chen, Liang-Chieh},
  booktitle =    {IEEE conference on computer vision and pattern recognition
                  (CVPR)},
  year =         2018
}

@misc{ren2022kroneckerfactored,
  title =        {Kronecker-factored Quasi-Newton Methods for Deep Learning},
  author =       {Yi Ren and Achraf Bahamou and Donald Goldfarb},
  year =         2022,
}

@article{arora2019exact,
  title =        {On exact computation with an infinitely wide neural net},
  author =       {Arora, Sanjeev and Du, Simon S and Hu, Wei and Li, Zhiyuan and
                  Salakhutdinov, Russ R and Wang, Ruosong},
  journal =      {Advances in neural information processing systems (NeurIPS)},
  year =         2019
}

@misc{biamonte2017tensor,
  title =        {Tensor Networks in a Nutshell},
  author =       {Jacob Biamonte and Ville Bergholm},
  year =         2017,
}

@book{goodfellow2016deep,
  title =        {Deep Learning},
  author =       {Ian Goodfellow and Yoshua Bengio and Aaron Courville},
  year =         2016
}

@article{olikier2023gauss,
  title =        {Gauss-Southwell type descent methods for low-rank matrix
                  optimization},
  author =       {Olikier, Guillaume and Uschmajew, Andr{\'e} and Vandereycken,
                  Bart},
  year =         2023
}

@article{modoranu2023error,
  title =        {Error Feedback Can Accurately Compress Preconditioners},
  author =       {Modoranu, Ionut-Vlad and Kalinov, Aleksei and Kurtic, Eldar
                  and Alistarh, Dan},
  year =         2023
}

@article{ritter2018online,
  title =        {Online structured laplace approximations for overcoming
                  catastrophic forgetting},
  author =       {Ritter, Hippolyt and Botev, Aleksandar and Barber, David},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2018
}

@inproceedings{huang2017densely,
  title =        {Densely connected convolutional networks},
  author =       {Huang, Gao and Liu, Zhuang and Van Der Maaten, Laurens and
                  Weinberger, Kilian Q},
  booktitle =    {IEEE conference on computer vision and pattern recognition
                  (CVPR)},
  year =         2017
}

@inproceedings{xie2017aggregated,
  title =        {Aggregated residual transformations for deep neural networks},
  author =       {Xie, Saining and Girshick, Ross and Doll{\'a}r, Piotr and Tu,
                  Zhuowen and He, Kaiming},
  booktitle =    {IEEE conference on computer vision and pattern recognition
                  (CVPR)},
  year =         2017
}

@article{zagoruyko2016wide,
  title =        {Wide residual networks},
  author =       {Zagoruyko, Sergey and Komodakis, Nikos},
  year =         2016
}

@article{damm2002complexity,
  title =        {The complexity of tensor calculus},
  author =       {Damm, Carsten and Holzer, Markus and McKenzie, Pierre},
  journal =      {computational complexity},
  year =         2002,
}

@inproceedings{vaswani2017attention,
  author =       {Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and
                  Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and
                  Kaiser, \L ukasz and Polosukhin, Illia},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  title =        {Attention is All you Need},
  year =         2017
}

@article{lecun1989backpropagation,
  author =       {LeCun, Y. and Boser, B. and Denker, J. S. and Henderson, D.
                  and Howard, R. E. and Hubbard, W. and Jackel, L. D.},
  journal =      {Neural Computation},
  title =        {Backpropagation Applied to Handwritten Zip Code Recognition},
  year =         1989,
}

@book{gates2021avoid,
  title =        {How to Avoid a Climate Disaster: The Solutions We Have and the
                  Breakthroughs We Need},
  author =       {Gates, Bill},
  year =         2021,
  publisher =    {Penguin Books Limited}
}

@article{levin2022effect,
  title =        {The effect of smooth parametrizations on nonconvex
                  optimization landscapes},
  author =       {Levin, Eitan and Kileel, Joe and Boumal, Nicolas},
  year =         2022
}

@article{matena2022merging,
  title =        {Merging models with fisher-weighted averaging},
  author =       {Matena, Michael S and Raffel, Colin A},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2022
}

@article{novikov2015tensorizing,
  title =        {Tensorizing neural networks},
  author =       {Novikov, Alexander and Podoprikhin, Dmitrii and Osokin, Anton
                  and Vetrov, Dmitry P},
  journal =      {Advances in neural information processing systems (NeurIPS)},
  year =         2015
}

@article{blacher2023efficient,
  title =        {Efficient and Portable Einstein Summation in SQL},
  author =       {Blacher, Mark and Klaus, Julien and Staudt, Christoph and
                  Laue, S{\"o}ren and Leis, Viktor and Giesen, Joachim},
  journal =      {ACM on Management of Data},
  year =         2023,
}

@inproceedings{klaus2023compiling,
  author =       "Klaus, Julien and Blacher, Mark and Giesen, Joachim",
  title =        "Compiling Tensor Expressions into Einsum",
  booktitle =    "International Conference on Computational Science (ICCS)",
  year =         2023,
}

@article{bauer1974computational,
  author =       {F. L. Bauer},
  journal =      {SIAM Journal on Numerical Analysis},
  title =        {Computational Graphs and Rounding Error},
  year =         1974
}

@inproceedings{dangel2024convolutions,
  title =        {Convolutions and More as Einsum: A Tensor Network Perspective
                  with Advances for Second-Order Methods},
  author =       {Dangel, Felix},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2024,
  keywords =     {mywork},
}

@inproceedings{cheng2022stochastic,
  title =        {Stochastic backpropagation: A memory efficient strategy for
                  training video models},
  author =       {Cheng, Feng and Xu, Mingze and Xiong, Yuanjun and Chen, Hao
                  and Li, Xinyu and Li, Wei and Xia, Wei},
  booktitle =    {IEEE/CVF Conference on Computer Vision and Pattern Recognition
                  (CVPR)},
  year =         2022
}

@inproceedings{chen2021actnn,
  title =        {ActNN: Reducing Training Memory Footprint via 2-Bit Activation
                  Compressed Training},
  author =       {Chen, Jianfei and Zheng, Lianmin and Yao, Zhewei and Wang,
                  Dequan and Stoica, Ion and Mahoney, Michael W and Gonzalez,
                  Joseph E},
  booktitle =    {International Conference on Machine Learning (ICLR)},
  year =         2021
}

@article{leventhal2011randomized,
  title =        {Randomized Hessian estimation and directional search},
  author =       {Leventhal, D and Lewis, AS},
  journal =      {Optimization},
  year =         2011,
}

@inproceedings{amari2019fisher,
  title =        {Fisher information and natural gradient learning in random
                  deep networks},
  author =       {Amari, Shun-ichi and Karakida, Ryo and Oizumi, Masafumi},
  booktitle =    {International Conference on Artificial Intelligence and
                  Statistics (AISTATS)},
  year =         2019,
}

@inproceedings{graham2018semantic,
  title =        {3d semantic segmentation with submanifold sparse convolutional
                  networks},
  author =       {Graham, Benjamin and Engelcke, Martin and Van Der Maaten,
                  Laurens},
  booktitle =    {IEEE conference on computer vision and pattern recognition
                  (CVPR)},
  year =         2018
}

@article{hall2023mental,
  title =        {A mental-health crisis is gripping science—toxic research
                  culture is to blame},
  author =       {Hall, Shannon},
  journal =      {Nature},
  year =         2023,
}

@article{graham2015sparse,
  title =        {Sparse 3D convolutional neural networks},
  author =       {Graham, Ben},
  year =         2015
}

@article{kanakagiri2023minimum,
  title =        {Minimum Cost Loop Nests for Contraction of a Sparse Tensor
                  with a Tensor Network},
  author =       {Kanakagiri, Raghavendra and Solomonik, Edgar},
  year =         2023
}

@misc{godbole2023deep,
  author =       {Varun Godbole and George E. Dahl and Justin Gilmer and
                  Christopher J. Shallue and Zachary Nado},
  title =        {Deep Learning Tuning Playbook},
  year =         2023,
}

@inproceedings{novikov2023few,
  title =        {Few-bit backward: Quantized gradients of activation functions
                  for memory footprint reduction},
  author =       {Novikov, Georgii Sergeevich and Bershatsky, Daniel and Gusak,
                  Julia and Shonenkov, Alex and Dimitrov, Denis Valerievich and
                  Oseledets, Ivan},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2023,
}

@inproceedings{pinson2023linear,
  title =        {Linear CNNs Discover the Statistical Structure of the Dataset
                  Using Only the Most Dominant Frequencies},
  author =       {Pinson, Hannah and Lenaerts, Joeri and Ginis, Vincent},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2023
}

@inproceedings{nikdan2023sparseprop,
  title =        {SparseProp: Efficient Sparse Backpropagation for Faster
                  Training of Neural Networks},
  author =       {Nikdan, Mahdi and Pegolotti, Tommaso and Iofinova, Eugenia and
                  Kurtic, Eldar and Alistarh, Dan},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2023
}

@inproceedings{muller2023achieving,
  title =        {Achieving High Accuracy with PINNs via Energy Natural Gradient
                  Descent},
  author =       {M{\"u}ller, Johannes and Zeinhofer, Marius},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2023
}

@misc{dahl2023benchmarking,
  title =        {Benchmarking Neural Network Training Algorithms},
  author =       {Dahl, George E and Schneider, Frank and Nado, Zachary and
                  Agarwal, Naman and Sastry, Chandramouli Shama and Hennig,
                  Philipp and Medapati, Sourabh and Eschenhagen, Runa and
                  Kasimbeg, Priya and Suo, Daniel and others},
  year =         2023
}

@article{andriushchenko2023modern,
  title =        {A modern look at the relationship between sharpness and
                  generalization},
  author =       {Andriushchenko, Maksym and Croce, Francesco and M{\"u}ller,
                  Maximilian and Hein, Matthias and Flammarion, Nicolas},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2023
}

@inproceedings{kwon2021asam,
  title =        {Asam: Adaptive sharpness-aware minimization for
                  scale-invariant learning of deep neural networks},
  author =       {Kwon, Jungmin and Kim, Jeongseop and Park, Hyunseo and Choi,
                  In Kwon},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2021,
}

@inproceedings{koh2017understanding,
  title =        {Understanding black-box predictions via influence functions},
  author =       {Koh, Pang Wei and Liang, Percy},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2017,
}

@misc{lin2021smooth,
  title =        {Introduction to Natural-gradient Descent, Part I: Smooth
                  Manifolds with the Fisher-Rao Metric},
  author =       {Lin, Wu and Nielsen, Frank and Khan, Mohammad Emtiyaz and
                  Schmidt, Mark},
  howpublished =
                  {\url{https://informationgeometryml.github.io/posts/2021/09/Geomopt01/}},
  year =         2021,
  note =         {Accessed: 2023-08-22}
}

@misc{lin2021derivation,
  title =        {Introduction to Natural-gradient Descent, Part II: Derivation
                  of natural-gradients},
  author =       {Lin, Wu and Nielsen, Frank and Khan, Mohammad Emtiyaz and
                  Schmidt, Mark},
  howpublished =
                  {\url{https://informationgeometryml.github.io/posts/2021/10/Geomopt02/}},
  year =         2021,
  note =         {Accessed: 2023-08-23}
}

@misc{lin2021invariance,
  title =        {Introduction to Natural-gradient Descent, Part III: Invariance
                  of Natural-Gradients},
  author =       {Lin, Wu and Nielsen, Frank and Khan, Mohammad Emtiyaz and
                  Schmidt, Mark},
  howpublished =
                  {\url{https://informationgeometryml.github.io/posts/2021/11/Geomopt03/}},
  year =         2021,
  note =         {Accessed: 2023-08-23}
}

@inproceedings{jahani2022doubly,
  title =        {Doubly Adaptive Scaled Algorithm for Machine Learning Using
                  Second-Order Information},
  author =       {Majid Jahani and Sergey Rusakov and Zheng Shi and Peter
                  Richt{\'a}rik and Michael W. Mahoney and Martin Takac},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2022,
}

@article{tan2022accelerating,
  title =        {Accelerating Sparse Convolution with Column Vector-Wise
                  Sparsity},
  author =       {Tan, Yijun and Han, Kai and Zhao, Kang and Yu, Xianzhi and Du,
                  Zidong and Chen, Yunji and Wang, Yunhe and Yao, Jun},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2022
}

@article{khajehabdollahi2023locally,
  title =        {Locally adaptive cellular automata for goal-oriented
                  self-organization},
  author =       {Khajehabdollahi, Sina and Giannakakis, Emmanouil and Buendia,
                  Victor and Martius, Georg and Levina, Anna},
  year =         2023
}

@misc{shi2023distributed,
  title =        {A Distributed Data-Parallel PyTorch Implementation of the
                  Distributed Shampoo Optimizer for Training Neural Networks
                  At-Scale},
  author =       {Hao-Jun Michael Shi and Tsung-Hsien Lee and Shintaro Iwasaki
                  and Jose Gallego-Posada and Zhijing Li and Kaushik Rangadurai
                  and Dheevatsa Mudigere and Michael Rabbat},
  year =         2023,
}

@article{grubisic2023looptune,
  title =        {LoopTune: Optimizing Tensor Computations with Reinforcement
                  Learning},
  author =       {Grubisic, Dejan and Wasti, Bram and Cummins, Chris and
                  Mellor-Crummey, John and Zlateski, Aleksandar},
  year =         2023
}

@article{pfau2020ab,
  title =        {Ab initio solution of the many-electron {S}chr{\"o}dinger
                  equation with deep neural networks},
  author =       {Pfau, David and Spencer, James S and Matthews, Alexander GDG
                  and Foulkes, W Matthew C},
  journal =      {Physical Review Research},
  year =         2020,
}

@inproceedings{izadi2020optimization,
  title =        {Optimization of graph neural networks with natural gradient
                  descent},
  author =       {Izadi, Mohammad Rasool and Fang, Yihao and Stevenson, Robert
                  and Lin, Lizhen},
  booktitle =    {IEEE International Conference on Big Data (IEEE BigData)},
  year =         2020,
}

@misc{turchetti2023decomposition,
  title =        {Decomposition of linear tensor transformations},
  author =       {Claudio Turchetti},
  year =         2023,
}

@article{hutchinson1989stochastic,
  author =       {Hutchinson, M.F.},
  year =         1989,
  title =        {A stochastic estimator of the trace of the influence matrix
                  for Laplacian smoothing splines},
  journal =      {Communication in Statistics---Simulation and Computation},
}

@article{jiang2021optimal,
  title =        {Optimal sketching for trace estimation},
  author =       {Jiang, Shuli and Pham, Hai and Woodruff, David and Zhang,
                  Richard},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2021
}

@inproceedings{kristiadi2023geometry,
  title =        {The Geometry of Neural Nets' Parameter Spaces Under
                  Reparametrization},
  author =       {Kristiadi, Agustinus and Dangel, Felix and Hennig, Philipp},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2023,
  keywords =     {mywork},
}

@inproceedings{bae2022amortized,
  author =       {Bae, Juhan and Vicol, Paul and HaoChen, Jeff Z. and Grosse,
                  Roger B},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  title =        {Amortized Proximal Optimization},
  year =         2022
}

@inproceedings{tran2022better,
  title =        {Better {SGD} using Second-order Momentum},
  author =       {Hoang Tran and Ashok Cutkosky},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2022,
}

@article{murray2019revisiting,
  title =        {Revisiting normalized gradient descent: Fast evasion of saddle
                  points},
  author =       {Murray, Ryan and Swenson, Brian and Kar, Soummya},
  journal =      {IEEE Transactions on Automatic Control},
  year =         2019,
}

@article{shukla2023randomized,
  title =        {Randomized Forward Mode of Automatic Differentiation for
                  Optimization Algorithms},
  author =       {Khemraj Shukla and Yeonjong Shin},
  year =         2023,
}

@article{turner2023introduction,
  title =        {An Introduction to Transformers},
  author =       {Turner, Richard E},
  year =         2023
}

@article{tropp2019streaming,
  title =        {Streaming Low-Rank Matrix Approximation with an Application to
                  Scientific Simulation},
  author =       {Joel A. Tropp and Alp Yurtsever and Madeleine Udell and Volkan
                  Cevher},
  year =         2019,
}

@article{foulkes2001quantum,
  title =        {Quantum {M}onte {C}arlo simulations of solids},
  author =       {Foulkes, William MC and Mitas, Lubos and Needs, RJ and
                  Rajagopal, Guna},
  journal =      {Reviews of Modern Physics},
  volume =       73,
  number =       1,
  pages =        33,
  year =         2001,
  publisher =    {APS}
}

@article{li2023forward,
  title =        {A computational framework for neural network-based variational
                  {M}onte {C}arlo with Forward {L}aplacian},
  author =       {Li, Ruichen and Ye, Haotian and Jiang, Du and Wen, Xuelan and
                  Wang, Chuwei and Li, Zhe and Li, Xiang and He, Di and Chen, Ji
                  and Ren, Weiluo and others},
  journal =      {Nature Machine Intelligence},
  year =         2024,
}

@article{koroko2022efficient,
  title =        {Efficient approximations of the fisher matrix in neural
                  networks using kronecker product singular value decomposition},
  author =       {Koroko, Abdoulaye and Anciaux-Sedrakian, Ani and Gharbia,
                  Ibtihel Ben and Gar{\`e}s, Val{\'e}rie and Haddou, Mounir and
                  Tran, Quang Huy},
  year =         2022
}

@inproceedings{eschenhagen2023kroneckerfactored,
  title =        {Kronecker-Factored Approximate Curvature for Modern Neural
                  Network Architectures},
  author =       {Runa Eschenhagen and Alexander Immer and Richard E. Turner and
                  Frank Schneider and Philipp Hennig},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2023,
}

@article{liu2023sophia,
  title =        {Sophia: A Scalable Stochastic Second-order Optimizer for
                  Language Model Pre-training},
  author =       {Liu, Hong and Li, Zhiyuan and Hall, David and Liang, Percy and
                  Ma, Tengyu},
  year =         2023
}

@article{fu2023flashfftconv,
  title =        {FlashFFTConv: Efficient Convolutions for Long Sequences with
                  Tensor Cores},
  author =       {Fu, Daniel Y and Kumbong, Hermann and Nguyen, Eric and R{\'e},
                  Christopher},
  year =         2023
}

@article{gray2022hyper,
  title =        {Hyper-optimized compressed contraction of tensor networks with
                  arbitrary geometry},
  author =       {Gray, Johnnie and Chan, Garnet Kin},
  year =         2022
}

@article{valiant2023matrix,
  title =        {Matrix Multiplication in Quadratic Time and Energy? Towards a
                  Fine-Grained Energy-Centric Church-Turing Thesis},
  author =       {Valiant, Gregory},
  year =         2023
}

@article{tan2021analytic,
  title =        {Analytic natural gradient updates for Cholesky factor in
                  Gaussian variational approximation},
  author =       {Tan, Linda SL},
  year =         2021
}

@misc{wolinski2023adapting,
  title =        {Adapting Newton's Method to Neural Networks through a Summary
                  of Higher-Order Derivatives},
  author =       {Pierre Wolinski},
  year =         2023,
}

@inproceedings{zhang2023optimal,
  title =        {Optimal shrinkage for distributed second-order optimization},
  author =       {Zhang, Fangzhao and Pilanci, Mert},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2023,
}

@article{tam2023merging,
  title =        {Merging by Matching Models in Task Subspaces},
  author =       {Tam, Derek and Bansal, Mohit and Raffel, Colin},
  year =         2023
}

@inproceedings{potapczynski2023cola,
  title =        {{CoLA: Exploiting Compositional Structure for Automatic and
                  Efficient Numerical Linear Algebra}},
  author =       {Andres Potapczynski and Marc Finzi and Geoff Pleiss and Andrew
                  Gordon Wilson},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2023
}

@article{wu2023cytnx,
  title =        {The Cytnx Library for Tensor Networks},
  author =       {Wu, Kai-Hsin and Lin, Chang-Teng and Hsu, Ke and Hung, Hao-Ti
                  and Schneider, Manuel},
  year =         2023
}

@inproceedings{perez2023training,
  title =        {Training and inference of large language models using 8-bit
                  floating point},
  author =       {Sergio Perez and Yan Zhang and James Briggs and Charlie Blake
                  and Josh Levy-Kramer and Paul Balanca and Carlo Luschi and
                  Stephen Barlow and Andrew Fitzgibbon},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS),
                  Workshop on Advancing Neural Network Training (WANT):
                  Computational Efficiency, Scalability, and Resource
                  Optimization},
  year =         2023,
}

@inproceedings{blake2023unit,
  author =       {Blake, Charlie and Orr, Douglas and Luschi, Carlo},
  title =        {Unit Scaling: Out-of-the-Box Low-Precision Training},
  year =         2023,
  booktitle =    {International Conference on Machine Learning (ICML)},
}

@article{singhal2023guess,
  title =        {How to guess a gradient},
  author =       {Singhal, Utkarsh and Cheung, Brian and Chandra, Kartik and
                  Ragan-Kelley, Jonathan and Tenenbaum, Joshua B and Poggio,
                  Tomaso A and Yu, Stella X},
  year =         2023
}

@article{desjardins2015natural,
  title =        {Natural neural networks},
  author =       {Desjardins, Guillaume and Simonyan, Karen and Pascanu, Razvan
                  and others},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2015
}

@inproceedings{khan2018fast,
  title =        {Fast and scalable bayesian deep learning by
                  weight-perturbation in adam},
  author =       {Khan, Mohammad and Nielsen, Didrik and Tangkaratt, Voot and
                  Lin, Wu and Gal, Yarin and Srivastava, Akash},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2018,
}

@inproceedings{bettencourt2019taylor,
  title =        {Taylor-mode automatic differentiation for higher-order
                  derivatives in {JAX}},
  author =       {Bettencourt, Jesse and Johnson, Matthew J and Duvenaud, David},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS);
                  Workhop on Program Transformations for ML},
  year =         2019
}

@article{ali2023traveling,
  title =        {Traveling Salesman Problem from a Tensor Networks Perspective},
  author =       {Ali, Alejandro Mata and Delgado, I{\~n}igo Perez and de
                  Leceta, Aitor Moreno Fdez},
  year =         2023
}

@article{team2023gemini,
  title =        {Gemini: A family of highly capable multimodal models},
  author =       {Team, Gemini and Anil, Rohan and Borgeaud, Sebastian and Wu,
                  Yonghui and Alayrac, Jean-Baptiste and Yu, Jiahui and Soricut,
                  Radu and Schalkwyk, Johan and Dai, Andrew M and Hauth, Anja
                  and others},
  year =         2023
}

@article{sun2024dynamical,
  title =        {Dynamical measure transport and neural {PDE} solvers for
                  sampling},
  author =       {Sun, Jingtong and Berner, Julius and Richter, Lorenz and
                  Zeinhofer, Marius and M{\"u}ller, Johannes and
                  Azizzadenesheli, Kamyar and Anandkumar, Anima},
  journal =      {arXiv preprint arXiv:2407.07873},
  year =         2024
}

@article{hu2024hutchinson,
  title={Hutchinson trace estimation for high-dimensional and high-order physics-informed neural networks},
  author={Hu, Zheyuan and Shi, Zekun and Karniadakis, George Em and Kawaguchi, Kenji},
  journal={Computer Methods in Applied Mechanics and Engineering},
  year={2024},
}

@misc{johnson2021taylor-mode,
  title =        {Taylor-mode higher-order automatic differentiation},
  author =       {Matthew J. Johnson and Jesse Bettencourt and Dougal Maclaurin
                  and David Duvenaud},
  year =         2021,
  note =
                  {\href{https://github.com/google/jax/files/6717197/jet.pdf}{Accessed
                  January 03, 2024}}
}

@inproceedings{tang2021skfac,
  title =        {Skfac: Training neural networks with faster kronecker-factored
                  approximate curvature},
  author =       {Tang, Zedong and Jiang, Fenlong and Gong, Maoguo and Li, Hao
                  and Wu, Yue and Yu, Fan and Wang, Zidong and Wang, Min},
  booktitle =    {IEEE Conference on Computer Vision and Pattern Recognition
                  (CVPR)},
  year =         2021
}

@software{dangel2022curvlinops,
  author =       {Felix Dangel},
  title =        {curvlinops: scipy linear operators for the {H}essian,
                  Fisher/GGN, and more in PyTorch},
  year =         2022,
  journal =      {GitHub repository (https://github.com/f-dangel/curvlinops)},
  keywords =     {mywork},
}

@article{goodbrake2023unnatural,
  title =        {Unnatural Algorithms in Machine Learning},
  author =       {Goodbrake, Christian},
  year =         2023
}

@inproceedings{absil2013extrinsic,
  title =        {An extrinsic look at the Riemannian {H}essian},
  author =       {Absil, P-A and Mahony, Robert and Trumpf, Jochen},
  booktitle =    {International conference on geometric science of information
                  (GSI)},
  year =         2013,
}

@book{boumal2023introduction,
  title =        {An introduction to optimization on smooth manifolds},
  author =       {Boumal, Nicolas},
  year =         2023,
  publisher =    {Cambridge University Press}
}

@article{chetlur2014cudnn,
  title =        {cudnn: Efficient primitives for deep learning},
  author =       {Chetlur, Sharan and Woolley, Cliff and Vandermersch, Philippe
                  and Cohen, Jonathan and Tran, John and Catanzaro, Bryan and
                  Shelhamer, Evan},
  year =         2014
}

@misc{grosse2023studying,
  title =        {Studying Large Language Model Generalization with Influence
                  Functions},
  author =       {Roger Grosse and Juhan Bae and Cem Anil and Nelson Elhage and
                  Alex Tamkin and Amirhossein Tajdini and Benoit Steiner and
                  Dustin Li and Esin Durmus and Ethan Perez and Evan Hubinger
                  and Kamilė Lukošiūtė and Karina Nguyen and Nicholas Joseph and
                  Sam McCandlish and Jared Kaplan and Samuel R. Bowman},
  year =         2023,
}

@inproceedings{benzing2022gradient,
  title =        {Gradient Descent on Neurons and its Link to Approximate
                  Second-order Optimization},
  author =       {Benzing, Frederik},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2022,
}

@misc{roulet2023dual,
  title =        {Dual Gauss-Newton Directions for Deep Learning},
  author =       {Vincent Roulet and Mathieu Blondel},
  year =         2023,
}

@misc{dao2022flashattention,
  title =        {FlashAttention: Fast and Memory-Efficient Exact Attention with
                  IO-Awareness},
  author =       {Tri Dao and Daniel Y. Fu and Stefano Ermon and Atri Rudra and
                  Christopher Ré},
  year =         2022,
}

@misc{dao2023flashattention2,
  title =        {FlashAttention-2: Faster Attention with Better Parallelism and
                  Work Partitioning},
  author =       {Tri Dao},
  year =         2023,
}

@inproceedings{blondel2022efficient,
  title =        {Efficient and Modular Implicit Differentiation},
  author =       {Mathieu Blondel and Quentin Berthet and marco cuturi and Roy
                  Frostig and Stephan Hoyer and Felipe Llinares-L{\'o}pez and
                  Fabian Pedregosa and Jean-Philippe Vert},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2022,
}

@misc{vlaar2022linear,
  title =        {What can linear interpolation of neural network loss
                  landscapes tell us?},
  author =       {Tiffany Vlaar and Jonathan Frankle},
  year =         2022,
}

@misc{sohldickstein2012natural,
  title =        {The Natural Gradient by Analogy to Signal Whitening, and
                  Recipes and Tricks for its Use},
  author =       {Jascha Sohl-Dickstein},
  year =         2012,
}

@misc{drissi2024secondorder,
  title =        {Second-order optimisation strategies for neural network
                  quantum states},
  author =       {M. Drissi and J. W. T. Keeble and J. Rozalén Sarmiento and A.
                  Rios},
  year =         2024,
}

@misc{dauphin2024neglected,
  title =        {Neglected Hessian component explains mysteries in Sharpness
                  regularization},
  author =       {Yann N. Dauphin and Atish Agarwala and Hossein Mobahi},
  year =         2024,
}

@misc{sun2023retentive,
  title =        {Retentive Network: A Successor to Transformer for Large
                  Language Models},
  author =       {Yutao Sun and Li Dong and Shaohan Huang and Shuming Ma and
                  Yuqing Xia and Jilong Xue and Jianyong Wang and Furu Wei},
  year =         2023,
}

@misc{ding2023longnet,
  title =        {LongNet: Scaling Transformers to 1,000,000,000 Tokens},
  author =       {Jiayu Ding and Shuming Ma and Li Dong and Xingxing Zhang and
                  Shaohan Huang and Wenhui Wang and Nanning Zheng and Furu Wei},
  year =         2023,
}

@article{henderson1981deriving,
  author =       {Henderson, H. V. and Searle, S. R.},
  title =        {On Deriving the Inverse of a Sum of Matrices},
  journal =      {SIAM Review},
  year =         1981,
}

@inproceedings{dharangutte2023tight,
  title =        {A Tight Analysis of Hutchinson's Diagonal Estimator},
  author =       {Dharangutte, Prathamesh and Musco, Christopher},
  booktitle =    {Symposium on Simplicity in Algorithms (SOSA)},
  year =         2023,
}

@article{soen2024tradeoffs,
  title =        {Tradeoffs of Diagonal Fisher Information Matrix Estimators},
  author =       {Soen, Alexander and Sun, Ke},
  year =         2024
}

@article{doikov2024spectral,
  title =        {Spectral Preconditioning for Gradient Methods on Graded
                  Non-convex Functions},
  author =       {Doikov, Nikita and Stich, Sebastian U and Jaggi, Martin},
  year =         2024
}

@inproceedings{frantar2021efficient,
  title =        {M-{FAC}: Efficient Matrix-Free Approximations of Second-Order
                  Information},
  author =       {Elias Frantar and Eldar Kurtic and Dan Alistarh},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2021,
}

@article{ba2016layer,
  title =        {Layer normalization},
  author =       {Ba, Jimmy Lei and Kiros, Jamie Ryan and Hinton, Geoffrey E},
  year =         2016
}

@article{ulyanov2016instance,
  title =        {Instance normalization: The missing ingredient for fast
                  stylization},
  author =       {Ulyanov, Dmitry and Vedaldi, Andrea and Lempitsky, Victor},
  year =         2016
}

@inproceedings{gatys2016image,
  title =        {Image style transfer using convolutional neural networks},
  author =       {Gatys, Leon A and Ecker, Alexander S and Bethge, Matthias},
  booktitle =    {IEEE Conference on Computer Vision and Pattern Recognition
                  (CVPR)},
  year =         2016
}

@inproceedings{zhao2024tuning,
  title =        {Tuning LayerNorm in Attention: Towards Efficient Multi-Modal
                  {LLM} Finetuning},
  author =       {Bingchen Zhao and Haoqin Tu and Chen Wei and Jieru Mei and
                  Cihang Xie},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2024,
}

@inproceedings{lee2023surgical,
  title =        {Surgical Fine-Tuning Improves Adaptation to Distribution
                  Shifts},
  author =       {Yoonho Lee and Annie S Chen and Fahim Tajwar and Ananya Kumar
                  and Huaxiu Yao and Percy Liang and Chelsea Finn},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2023,
}

@inproceedings{hu2022lora,
  title =        {Lo{RA}: Low-Rank Adaptation of Large Language Models},
  author =       {Edward J Hu and yelong shen and Phillip Wallis and Zeyuan
                  Allen-Zhu and Yuanzhi Li and Shean Wang and Lu Wang and Weizhu
                  Chen},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2022,
}

@inproceedings{goodfellow2015explaining,
  title =        {Explaining and harnessing adversarial examples},
  author =       {Goodfellow, Ian J and Shlens, Jonathon and Szegedy, Christian},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2015
}

@article{hubara2018quantized,
  title =        {Quantized neural networks: Training neural networks with low
                  precision weights and activations},
  author =       {Hubara, Itay and Courbariaux, Matthieu and Soudry, Daniel and
                  El-Yaniv, Ran and Bengio, Yoshua},
  journal =      {Journal of Machine Learning Research (JMLR)},
  year =         2018
}

@article{li2017training,
  title =        {Training quantized nets: A deeper understanding},
  author =       {Li, Hao and De, Soham and Xu, Zheng and Studer, Christoph and
                  Samet, Hanan and Goldstein, Tom},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2017
}

@misc{nagel2021white,
  title =        {A White Paper on Neural Network Quantization},
  author =       {Markus Nagel and Marios Fournarakis and Rana Ali Amjad and
                  Yelysei Bondarenko and Mart van Baalen and Tijmen Blankevoort},
  year =         2021,
}

@misc{ren2021zerooffload,
  title =        {ZeRO-Offload: Democratizing Billion-Scale Model Training},
  author =       {Jie Ren and Samyam Rajbhandari and Reza Yazdani Aminabadi and
                  Olatunji Ruwase and Shuangyan Yang and Minjia Zhang and Dong
                  Li and Yuxiong He},
  year =         2021,
}

@article{webber2022rayleigh,
  title =        {Rayleigh-{G}auss-{N}ewton optimization with enhanced sampling for
                  variational {M}onte {C}arlo},
  author =       {Webber, Robert J and Lindsey, Michael},
  journal =      {Physical Review Research},
  year =         2022,
}

@inproceedings{lin2024automatic,
  title =        {Automatic Functional Differentiation in {JAX}},
  author =       {Min Lin},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2024,
}

@article{baston2022stochastic,
  title =        {Stochastic diagonal estimation: probabilistic bounds and an
                  improved algorithm},
  author =       {Baston, Robert A and Nakatsukasa, Yuji},
  year =         2022
}

@article{ghojogh2019eigenvalue,
  title =        {Eigenvalue and generalized eigenvalue problems: Tutorial},
  author =       {Ghojogh, Benyamin and Karray, Fakhri and Crowley, Mark},
  year =         2019
}

@inproceedings{tucker2005validated,
  title =        {Validated numerics for pedestrians},
  author =       {Tucker, Warwick},
  booktitle =    {European congress of mathematics},
  year =         2005
}

@inproceedings{dagreou2024how,
  author =       {Dagréou, Mathieu and Ablin, Pierre and Vaiter, Samuel and
                  Moreau, Thomas},
  title =        {How to compute {H}essian-vector products?},
  booktitle =    {International Conference on Learning Representations (ICLR)
                  Blogposts},
  year =         2024,
}

@article{arya2022automatic,
  title =        {Automatic differentiation of programs with discrete
                  randomness},
  author =       {Arya, Gaurav and Schauer, Moritz and Sch{\"a}fer, Frank and
                  Rackauckas, Christopher},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2022
}

@inproceedings{hui2021evaluation,
  title =        {Evaluation of neural architectures trained with square loss vs
                  cross-entropy in classification tasks},
  author =       {Like Hui and Mikhail Belkin},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2021,
}

@article{yu2018deep,
  title =        {The deep Ritz method: a deep learning-based numerical
                  algorithm for solving variational problems},
  author =       {Yu, Bing and Weinan, E},
  journal =      {Communications in Mathematics and Statistics},
  year =         2018,
}

}

@inproceedings{li2024dof,
  title =        {{DOF}: Accelerating High-order Differential Operators with
                  Forward Propagation},
  author =       {Ruichen Li and Chuwei Wang and Haotian Ye and Di He and Liwei
                  Wang},
  booktitle =    {International Conference on Learning Representations (ICLR),
                  Workshop on AI4DifferentialEquations In Science},
  year =         2024,
}

@article{clarke2023adam,
  title =        {Adam through a Second-Order Lens},
  author =       {Clarke, Ross M and Su, Baiyu and Hern{\'a}ndez-Lobato,
                  Jos{\'e} Miguel},
  year =         2023
}

@misc{micikevicius2017mixed,
  title =        {Mixed precision training},
  author =       {Micikevicius, Paulius and Narang, Sharan and Alben, Jonah and
                  Diamos, Gregory and Elsen, Erich and Garcia, David and
                  Ginsburg, Boris and Houston, Michael and Kuchaiev, Oleksii and
                  Venkatesh, Ganesh and others},
  year =         2017
}

@inproceedings{belouadi2024automatikz,
  title =        {AutomaTikZ: Text-Guided Synthesis of Scientific Vector
                  Graphics with TikZ},
  author =       {Jonas Belouadi and Anne Lauscher and Steffen Eger},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2024,
}

@misc{zhao2024galore,
  title =        {GaLore: Memory-Efficient LLM Training by Gradient Low-Rank
                  Projection},
  author =       {Jiawei Zhao and Zhenyu Zhang and Beidi Chen and Zhangyang Wang
                  and Anima Anandkumar and Yuandong Tian},
  year =         2024,
}

@inproceedings{lin2024structured,
  title =        {Structured Inverse-Free Natural Gradient Descent:
                  Memory-Efficient \& Numerically-Stable {KFAC}},
  author =       {Wu Lin and Felix Dangel and Runa Eschenhagen and Kirill
                  Neklyudov and Agustinus Kristiadi and Richard E. Turner and
                  Alireza Makhzani},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2024,
  keywords =     {mywork},
}

@inproceedings{lin2024can,
  title =        {Can We Remove the Square-Root in Adaptive Gradient Methods? A
                  Second-Order Perspective},
  author =       {Wu Lin and Felix Dangel and Runa Eschenhagen and Juhan Bae and
                  Richard E. Turner and Alireza Makhzani},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2024,
  keywords =     {mywork},
}

@article{wu2024improved,
  title =        {An Improved Empirical Fisher Approximation for Natural
                  Gradient Descent},
  author =       {Wu, Xiaodong and Yu, Wenyi and Zhang, Chao and Woodland,
                  Philip},
  year =         2024
}

@article{ma2024approximate,
  title =        {Approximate Contraction of Arbitrary Tensor Networks with a
                  Flexible and Efficient Density Matrix Algorithm},
  author =       {Ma, Linjian and Fishman, Matthew and Stoudenmire, Miles and
                  Solomonik, Edgar},
  year =         2024
}

@article{goldshlager2024kaczmarz,
  title =        {A Kaczmarz-inspired approach to accelerate the optimization of
                  neural network wavefunctions},
  author =       {Goldshlager, Gil and Abrahamsen, Nilin and Lin, Lin},
  year =         2024
}

@inproceedings{ament2022scalable,
  title =        {Scalable first-order bayesian optimization via structured
                  automatic differentiation},
  author =       {Ament, Sebastian E and Gomes, Carla P},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2022,
}

@inproceedings{zampini2024petscml,
  title =        {PETScML: Second-order solvers for training regression problems
                  in Scientific Machine Learning},
  author =       {Zampini, Stefano and Zerbinati, Umberto and Turkyyiah, George
                  and Keyes, David},
  booktitle =    {Platform for Advanced Scientific Computing Conference (PASC)},
  year =         2024
}

@article{donatella2024thermodynamic,
  title =        {Thermodynamic Natural Gradient Descent},
  author =       {Donatella, Kaelan and Duffield, Samuel and Aifer, Maxwell and
                  Melanson, Denis and Crooks, Gavin and Coles, Patrick J},
  year =         2024
}

@article{wang2023have,
  title =        {What have we learned from OpenReview?},
  author =       {Wang, Gang and Peng, Qi and Zhang, Yanfeng and Zhang,
                  Mingyang},
  journal =      {World Wide Web},
  year =         2023,
}

@article{chen2018closing,
  title =        {Closing the generalization gap of adaptive gradient methods in
                  training deep neural networks},
  author =       {Chen, Jinghui and Zhou, Dongruo and Tang, Yiqi and Yang, Ziyan
                  and Cao, Yuan and Gu, Quanquan},
  year =         2018
}

@misc{agarwal2022learning,
  title =        {Learning Rate Grafting: Transferability of Optimizer Tuning},
  author =       {Naman Agarwal and Rohan Anil and Elad Hazan and Tomer Koren
                  and Cyril Zhang},
  year =         2022,
}

@incollection{martens2012training,
  title =        {Training deep and recurrent networks with hessian-free
                  optimization},
  author =       {Martens, James and Sutskever, Ilya},
  booktitle =    {Neural Networks: Tricks of the Trade},
  year =         2012,
}

@article{van2023invariance,
  title =        {Invariance properties of the natural gradient in
                  overparametrised systems},
  author =       {van Oostrum, Jesse and M{\"u}ller, Johannes and Ay, Nihat},
  journal =      {Information geometry},
  year =         2023,
}

@misc{loshchilov2019decoupledweightdecayregularization,
  title =        {Decoupled Weight Decay Regularization},
  author =       {Ilya Loshchilov and Frank Hutter},
  year =         2019,
}

@inproceedings{elsayed2024revisiting,
  title =        {Revisiting Scalable Hessian Diagonal Approximations for
                  Applications in Reinforcement Learning},
  author =       {Mohamed Elsayed and Homayoon Farrahi and Felix Dangel and A.
                  Rupam Mahmood},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2024,
  keywords =     {mywork},
}

@misc{orabona2020maybe,
  author =       {Orabona, Francesco},
  title =        {Neural Networks (Maybe) Evolved to Make Adam the Best
                  Optimizer},
  year =         2019,
  note =
                  {\href{https://parameterfree.com/2020/12/06/neural-network-maybe-evolved-to-make-adam-the-best-optimizer/}{Blogpost},
                  accessed 2024-08-30},
}

@misc{maslej2024artificial,
  title =        {Artificial Intelligence Index Report 2024},
  author =       {Nestor Maslej and Loredana Fattorini and Raymond Perrault and
                  Vanessa Parli and Anka Reuel and Erik Brynjolfsson and John
                  Etchemendy and Katrina Ligett and Terah Lyons and James
                  Manyika and Juan Carlos Niebles and Yoav Shoham and Russell
                  Wald and Jack Clark},
  year =         2024,
}

@misc{openai2024gpt4,
  title =        {GPT-4 Technical Report},
  author =       {OpenAI and Josh Achiam and Steven Adler and Sandhini Agarwal
                  and Lama Ahmad and Ilge Akkaya and Florencia Leoni Aleman and
                  Diogo Almeida and Janko Altenschmidt and Sam Altman and
                  Shyamal Anadkat and Red Avila and Igor Babuschkin and Suchir
                  Balaji and Valerie Balcom and Paul Baltescu and Haiming Bao
                  and Mohammad Bavarian and Jeff Belgum and Irwan Bello and Jake
                  Berdine and Gabriel Bernadett-Shapiro and Christopher Berner
                  and Lenny Bogdonoff and Oleg Boiko and Madelaine Boyd and
                  Anna-Luisa Brakman and Greg Brockman and Tim Brooks and Miles
                  Brundage and Kevin Button and Trevor Cai and Rosie Campbell
                  and Andrew Cann and Brittany Carey and Chelsea Carlson and
                  Rory Carmichael and Brooke Chan and Che Chang and Fotis
                  Chantzis and Derek Chen and Sully Chen and Ruby Chen and Jason
                  Chen and Mark Chen and Ben Chess and Chester Cho and Casey Chu
                  and Hyung Won Chung and Dave Cummings and Jeremiah Currier and
                  Yunxing Dai and Cory Decareaux and Thomas Degry and Noah
                  Deutsch and Damien Deville and Arka Dhar and David Dohan and
                  Steve Dowling and Sheila Dunning and Adrien Ecoffet and Atty
                  Eleti and Tyna Eloundou and David Farhi and Liam Fedus and
                  Niko Felix and Simón Posada Fishman and Juston Forte and
                  Isabella Fulford and Leo Gao and Elie Georges and Christian
                  Gibson and Vik Goel and Tarun Gogineni and Gabriel Goh and
                  Rapha Gontijo-Lopes and Jonathan Gordon and Morgan Grafstein
                  and Scott Gray and Ryan Greene and Joshua Gross and Shixiang
                  Shane Gu and Yufei Guo and Chris Hallacy and Jesse Han and
                  Jeff Harris and Yuchen He and Mike Heaton and Johannes
                  Heidecke and Chris Hesse and Alan Hickey and Wade Hickey and
                  Peter Hoeschele and Brandon Houghton and Kenny Hsu and Shengli
                  Hu and Xin Hu and Joost Huizinga and Shantanu Jain and Shawn
                  Jain and Joanne Jang and Angela Jiang and Roger Jiang and
                  Haozhun Jin and Denny Jin and Shino Jomoto and Billie Jonn and
                  Heewoo Jun and Tomer Kaftan and Łukasz Kaiser and Ali Kamali
                  and Ingmar Kanitscheider and Nitish Shirish Keskar and Tabarak
                  Khan and Logan Kilpatrick and Jong Wook Kim and Christina Kim
                  and Yongjik Kim and Jan Hendrik Kirchner and Jamie Kiros and
                  Matt Knight and Daniel Kokotajlo and Łukasz Kondraciuk and
                  Andrew Kondrich and Aris Konstantinidis and Kyle Kosic and
                  Gretchen Krueger and Vishal Kuo and Michael Lampe and Ikai Lan
                  and Teddy Lee and Jan Leike and Jade Leung and Daniel Levy and
                  Chak Ming Li and Rachel Lim and Molly Lin and Stephanie Lin
                  and Mateusz Litwin and Theresa Lopez and Ryan Lowe and
                  Patricia Lue and Anna Makanju and Kim Malfacini and Sam
                  Manning and Todor Markov and Yaniv Markovski and Bianca Martin
                  and Katie Mayer and Andrew Mayne and Bob McGrew and Scott
                  Mayer McKinney and Christine McLeavey and Paul McMillan and
                  Jake McNeil and David Medina and Aalok Mehta and Jacob Menick
                  and Luke Metz and Andrey Mishchenko and Pamela Mishkin and
                  Vinnie Monaco and Evan Morikawa and Daniel Mossing and Tong Mu
                  and Mira Murati and Oleg Murk and David Mély and Ashvin Nair
                  and Reiichiro Nakano and Rajeev Nayak and Arvind Neelakantan
                  and Richard Ngo and Hyeonwoo Noh and Long Ouyang and Cullen
                  O'Keefe and Jakub Pachocki and Alex Paino and Joe Palermo and
                  Ashley Pantuliano and Giambattista Parascandolo and Joel
                  Parish and Emy Parparita and Alex Passos and Mikhail Pavlov
                  and Andrew Peng and Adam Perelman and Filipe de Avila Belbute
                  Peres and Michael Petrov and Henrique Ponde de Oliveira Pinto
                  and Michael and Pokorny and Michelle Pokrass and Vitchyr H.
                  Pong and Tolly Powell and Alethea Power and Boris Power and
                  Elizabeth Proehl and Raul Puri and Alec Radford and Jack Rae
                  and Aditya Ramesh and Cameron Raymond and Francis Real and
                  Kendra Rimbach and Carl Ross and Bob Rotsted and Henri Roussez
                  and Nick Ryder and Mario Saltarelli and Ted Sanders and
                  Shibani Santurkar and Girish Sastry and Heather Schmidt and
                  David Schnurr and John Schulman and Daniel Selsam and Kyla
                  Sheppard and Toki Sherbakov and Jessica Shieh and Sarah Shoker
                  and Pranav Shyam and Szymon Sidor and Eric Sigler and Maddie
                  Simens and Jordan Sitkin and Katarina Slama and Ian Sohl and
                  Benjamin Sokolowsky and Yang Song and Natalie Staudacher and
                  Felipe Petroski Such and Natalie Summers and Ilya Sutskever
                  and Jie Tang and Nikolas Tezak and Madeleine B. Thompson and
                  Phil Tillet and Amin Tootoonchian and Elizabeth Tseng and
                  Preston Tuggle and Nick Turley and Jerry Tworek and Juan
                  Felipe Cerón Uribe and Andrea Vallone and Arun Vijayvergiya
                  and Chelsea Voss and Carroll Wainwright and Justin Jay Wang
                  and Alvin Wang and Ben Wang and Jonathan Ward and Jason Wei
                  and CJ Weinmann and Akila Welihinda and Peter Welinder and
                  Jiayi Weng and Lilian Weng and Matt Wiethoff and Dave Willner
                  and Clemens Winter and Samuel Wolrich and Hannah Wong and
                  Lauren Workman and Sherwin Wu and Jeff Wu and Michael Wu and
                  Kai Xiao and Tao Xu and Sarah Yoo and Kevin Yu and Qiming Yuan
                  and Wojciech Zaremba and Rowan Zellers and Chong Zhang and
                  Marvin Zhang and Shengjia Zhao and Tianhao Zheng and Juntang
                  Zhuang and William Zhuk and Barret Zoph},
  year =         2024,
}

@inproceedings{bhatia2024lowering,
  title =        {Lowering PyTorch's Memory Consumption for Selective
                  Differentiation},
  author =       {Samarth Bhatia and Felix Dangel},
  booktitle =    {International Conference on Machine Learning (ICML), 2nd
                  Workshop on Advancing Neural Network Training (WANT):
                  Computational Efficiency, Scalability, and Resource
                  Optimization},
  year =         2024,
  keywords =     {mywork},
}

@inproceedings{dangel2024kroneckerfactored,
  title =        {Kronecker-Factored Approximate Curvature for Physics-Informed
                  Neural Networks},
  author =       {Felix Dangel and Johannes Müller and Marius Zeinhofer},
  year =         2024,
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  keywords =     {mywork},
}

@misc{gupta2018shampoo,
  title =        {Shampoo: Preconditioned Stochastic Tensor Optimization},
  author =       {Vineet Gupta and Tomer Koren and Yoram Singer},
  year =         2018,
}

@inproceedings{agarwal2019efficient,
  title =        {Efficient full-matrix adaptive regularization},
  author =       {Agarwal, Naman and Bullins, Brian and Chen, Xinyi and Hazan,
                  Elad and Singh, Karan and Zhang, Cyril and Zhang, Yi},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2019,
}

@misc{xi2023trainingtransformers4bitintegers,
  title =        {Training Transformers with 4-bit Integers},
  author =       {Haocheng Xi and Changhao Li and Jianfei Chen and Jun Zhu},
  year =         2023,
}

@article{jnini2024gauss,
  title =        {Gauss-Newton Natural Gradient Descent for Physics-Informed
                  Computational Fluid Dynamics},
  author =       {Jnini, Anas and Vella, Flavio and Zeinhofer, Marius},
  year =         2024
}

@article{cobb2024second,
  title =        {Second-Order Forward-Mode Automatic Differentiation for
                  Optimization},
  author =       {Cobb, Adam D and Baydin, At{\i}l{\i}m G{\"u}ne{\c{s}} and
                  Pearlmutter, Barak A and Jha, Susmit},
  year =         2024
}

@article{nakatsukasa2020fast,
  title =        {Fast and stable randomized low-rank matrix approximation},
  author =       {Nakatsukasa, Yuji},
  year =         2020
}

@inproceedings{doumeche2024physics,
  title =        {Physics-informed machine learning as a kernel method},
  author =       {Doum{\`e}che, Nathan and Bach, Francis and Biau, G{\'e}rard
                  and Boyer, Claire},
  booktitle =    {Conference on Learning Theory (COLT)},
  year =         2024,
}

@article{korbit2024incremental,
  title =        {Incremental Gauss-Newton Descent for Machine Learning},
  author =       {Korbit, Mikalai and Zanon, Mario},
  year =         2024
}

@article{sitzmann2020implicit,
  title =        {Implicit neural representations with periodic activation
                  functions},
  author =       {Sitzmann, Vincent and Martel, Julien and Bergman, Alexander
                  and Lindell, David and Wetzstein, Gordon},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2020
}

@article{morwani2024new,
  title =        {A New Perspective on Shampoo's Preconditioner},
  author =       {Morwani, Depen and Shapira, Itai and Vyas, Nikhil and Malach,
                  Eran and Kakade, Sham and Janson, Lucas},
  year =         2024
}

@article{wang2024bit,
  title =        {4-bit Shampoo for Memory-Efficient Network Training},
  author =       {Wang, Sike and Li, Jia and Zhou, Pan and Huang, Hua},
  year =         2024
}

@article{song2024does,
  title =        {Does SGD really happen in tiny subspaces?},
  author =       {Song, Minhak and Ahn, Kwangjun and Yun, Chulhee},
  year =         2024
}

@inproceedings{geiping2022stochastic,
  title =        {Stochastic Training is Not Necessary for Generalization},
  author =       {Jonas Geiping and Micah Goldblum and Phil Pope and Michael
                  Moeller and Tom Goldstein},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2022,
}

@article{roosta2015improved,
  title =        {Improved bounds on sample size for implicit matrix trace
                  estimators},
  author =       {Roosta-Khorasani, Farbod and Ascher, Uri},
  journal =      {Foundations of Computational Mathematics (FoCM)},
  year =         2015,
}

@article{cortinovis2022randomized,
  title =        {On randomized trace estimates for indefinite matrices with an
                  application to determinants},
  author =       {Cortinovis, Alice and Kressner, Daniel},
  journal =      {Foundations of Computational Mathematics (FoCM},
  year =         2022,
}

@inproceedings{ishikawa2024on,
  title =        {On the Parameterization of Second-Order Optimization Effective
                  towards the Infinite Width},
  author =       {Satoki Ishikawa and Ryo Karakida},
  booktitle =    {International Conference on Learning Representations (ICLR)},
  year =         2024,
}

@article{crawshaw2022robustness,
  title =        {Robustness to unbounded smoothness of generalized signsgd},
  author =       {Crawshaw, Michael and Liu, Mingrui and Orabona, Francesco and
                  Zhang, Wei and Zhuang, Zhenxun},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2022
}

@article{han2018solving,
  author =       {Han, Jiequn and Jentzen, Arnulf and E, Weinan},
  title =        {Solving high-dimensional partial differential equations using
                  deep learning},
  journal =      {Proceedings of the National Academy of Sciences (PNAS)},
  year =         2018,
}

@article{dalcin2011parallel,
  title =        {Parallel distributed computing using Python},
  journal =      {Advances in Water Resources},
  year =         2011,
  author =       {Lisandro D. Dalcin and Rodrigo R. Paz and Pablo A. Kler and
                  Alejandro Cosimo},
}

@article{balay2019petsc,
  title =        {PETSc users manual},
  author =       {Balay, Satish and Abhyankar, Shrirang and Adams, Mark and
                  Brown, Jed and Brune, Peter and Buschelman, Kris and Dalcin,
                  Lisandro and Dener, Alp and Eijkhout, Victor and Gropp,
                  William and others},
  year =         2019,
}

@book{lehoucq1998arpack,
  title =        {ARPACK users' guide: solution of large-scale eigenvalue
                  problems with implicitly restarted Arnoldi methods},
  author =       {Lehoucq, Richard B and Sorensen, Danny C and Yang, Chao},
  year =         1998,
  publisher =    {SIAM}
}

@article{balles2020geometry,
  title =        {The geometry of sign gradient descent},
  author =       {Balles, Lukas and Pedregosa, Fabian and Roux, Nicolas Le},
  year =         2020
}

@article{bekas2007estimator,
  title =        {An estimator for the diagonal of a matrix},
  journal =      {Applied Numerical Mathematics},
  year =         2007,
  author =       {C. Bekas and E. Kokiopoulou and Y. Saad},
}

@misc{golmant2018hessian,
  author =       {Golmant, Noah and Yao, Zhewei and Gholami, Amir and Mahoney,
                  Michael and Gonzalez, Joseph},
  title =        {pytorch-hessian-eigenthings: efficient PyTorch Hessian
                  eigendecomposition},
  year =         2018,
}

@article{gardner2018gpytorch,
  title =        {GPytorch: Blackbox matrix-matrix gaussian process inference
                  with gpu acceleration},
  author =       {Gardner, Jacob and Pleiss, Geoff and Weinberger, Kilian Q and
                  Bindel, David and Wilson, Andrew G},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2018
}

@book{hestenes1952methods,
  title =        {Methods of conjugate gradients for solving linear systems},
  author =       {Hestenes, Magnus Rudolph and Stiefel, Eduard and others},
  year =         1952,
}

@article{fong2010lsmr,
  title =        {LSMR: An iterative algorithm for sparse least-squares
                  problems: Systems Optimization Laboratory},
  author =       {Fong, David and Saunders, Michael},
  year =         2010
}

@article{srivastava2014dropout,
  author =       {Nitish Srivastava and Geoffrey Hinton and Alex Krizhevsky and
                  Ilya Sutskever and Ruslan Salakhutdinov},
  title =        {Dropout: A Simple Way to Prevent Neural Networks from
                  Overfitting},
  journal =      {Journal of Machine Learning Research (JMLR)},
  year =         2014,
}

@misc{reid2024gemini,
  title =        {Gemini 1.5: Unlocking multimodal understanding across millions
                  of tokens of context},
  author =       {Reid, Machel and Savinov, Nikolay and Teplyashin, Denis and
                  Lepikhin, Dmitry and Lillicrap, Timothy and Alayrac,
                  Jean-baptiste and Soricut, Radu and Lazaridou, Angeliki and
                  Firat, Orhan and Schrittwieser, Julian and others},
  year =         2024
}

@misc{lin2024fast,
  title =        {Fast Fractional Natural Gradient Descent Using Learnable
                  Spectral Factorizations},
  year =         2024,
  author =       {Wu Lin and Felix Dangel and Runa Eschenhagen and Juhan Bae and
                  Richard E. Turner and Roger B. Grosse},
  keywords =     {mywork},
}

@misc{tatzel2024debiasing,
  title =        {Debiasing Mini-Batch Quadratics for Applications in Deep
                  Learning},
  author =       {Lukas Tatzel and Bálint Mucsányi and Osane Hackel and Philipp
                  Hennig},
  year =         2024,
}

@article{rabbani2024conv_einsum,
  title =        {conv\_einsum: A Framework for Representation and Fast
                  Evaluation of Multilinear Operations in Convolutional
                  Tensorial Neural Networks},
  author =       {Rabbani, Tahseen and Su, Jiahao and Liu, Xiaoyu and Chan,
                  David and Sangston, Geoffrey and Huang, Furong},
  year =         2024
}

@article{mlodozeniec2024influence,
  title =        {Influence Functions for Scalable Data Attribution in Diffusion
                  Models},
  author =       {Mlodozeniec, Bruno and Eschenhagen, Runa and Bae, Juhan and
                  Immer, Alexander and Krueger, David and Turner, Richard},
  year =         2024
}

@article{mueller2024fast,
  title =        {Fast Deep Hedging with Second-Order Optimization},
  author =       {Mueller, Konrad and Akkari, Amira and Gonon, Lukas and Wood,
                  Ben},
  year =         2024
}

@article{weber2008direct,
  title =        {Direct energy functional minimization under orthogonality
                  constraints},
  author =       {Weber, Valery and VandeVondele, Joost and Hutter, J{\"u}rg and
                  Niklasson, Anders},
  journal =      {The Journal of Chemical Physics (JCP)},
  year =         2008,
}

@article{kasim2022dqc,
  title =        {DQC: A Python program package for differentiable quantum
                  chemistry},
  author =       {Kasim, Muhammad F and Lehtola, Susi and Vinko, Sam M},
  journal =      {The Journal of Chemical Physics (JCP)},
  year =         2022,
}

@article{yoshikawa2022automatic,
  title =        {Automatic differentiation for the direct minimization approach
                  to the Hartree--Fock method},
  author =       {Yoshikawa, Naruki and Sumita, Masato},
  journal =      {The Journal of Physical Chemistry A (JPC A)},
  year =         2022,
}

@article{helal2024mess,
  title =        {MESS: Modern Electronic Structure Simulations},
  author =       {Helal, Hatem and Fitzgibbon, Andrew},
  year =         2024
}

@article{mathiasen2024reducing,
  title =        {Reducing the cost of quantum chemical data by backpropagating
                  through density functional theory},
  author =       {Mathiasen, Alexander and Helal, Hatem and Balanca, Paul and
                  Krzywaniak, Adam and Parviz, Ali and Hvilsh{\o}j, Frederik and
                  Banaszewski, Blazej and Luschi, Carlo and Fitzgibbon, Andrew
                  William},
  year =         2024
}

@inproceedings{vary2024optimization,
  title =        {Optimization without Retraction on the Random Generalized
                  Stiefel Manifold},
  author =       {Vary, Simon and Ablin, Pierre and Gao, Bin and Absil,
                  Pierre-Antoine},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2024,
}

@article{toulouse2007optimization,
  title =        {Optimization of quantum {M}onte {C}arlo wave functions by energy
                  minimization},
  author =       {Toulouse, Julien and Umrigar, Cyrus J},
  journal =      {The Journal of Chemical Physics (JCP)},
  year =         2007,
}

@article{neuscamman2012optimizing,
  title =        {Optimizing large parameter sets in variational quantum Monte
                  Carlo},
  author =       {Neuscamman, Eric and Umrigar, CJ and Chan, Garnet Kin-Lic},
  journal =      {Physical Review B},
  year =         2012,
}

@article{zhao2024theoretical,
  title =        {Theoretical characterisation of the Gauss-Newton conditioning
                  in Neural Networks},
  author =       {Zhao, Jim and Singh, Sidak Pal and Lucchi, Aurelien},
  year =         2024
}

@article{hao2024newton,
  title =        {Newton Informed Neural Operator for Solving Nonlinear Partials
                  Differential Equations},
  author =       {Hao, Wenrui and Liu, Xinliang and Yang, Yahong},
  year =         2024,
}

@article{he2024deferred,
  title =        {Deferred Poisoning: Making the Model More Vulnerable via
                  Hessian Singularization},
  author =       {He, Yuhao and Tian, Jinyu and Zheng, Xianwei and Dong, Li and
                  Li, Yuanman and Zhang, Leo Yu and Zhou, Jiantao},
  year =         2024
}

@inproceedings{buffelli2024exact,
  title =        {Exact, Tractable Gauss-Newton Optimization in Deep Reversible
                  Architectures Reveal Poor Generalization},
  author =       {Davide Buffelli and Jamie McGowan and Wangkun Xu and Alexandru
                  Cioba and Da-shan Shiu and Guillaume Hennequin and Alberto
                  Bernacchia},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2024,
}

@inproceedings{liu2024layer,
  title =        {A Layer-Wise Natural Gradient Optimizer for Training Deep
                  Neural Networks},
  author =       {Liu, Xiaolei and Li, Shaoshuai and Gao, Kaixin and Wang,
                  Binfeng},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2024,
}

@article{lawton2024learning,
  title =        {Learning Morphisms with Gauss-Newton Approximation for Growing
                  Networks},
  author =       {Lawton, Neal and Galstyan, Aram and Steeg, Greg Ver},
  year =         2024,
}

@inproceedings{blacher2024einsum,
  title =        {Einsum Benchmark: Enabling the Development of Next-Generation
                  Tensor Execution Engines},
  author =       {Blacher, Mark and Staudt, Christoph and Klaus, Julien and
                  Wenig, Maurice and Merk, Niklas and Breuer, Alexander and
                  Engel, Max and Laue, S{\"o}ren and Giesen, Joachim},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS),
                  Systems Datasets and Benchmarks Track},
  year =         2024,
}

@article{wang2019backpropagation,
  title =        {Backpropagation-friendly eigendecomposition},
  author =       {Wang, Wei and Dang, Zheng and Hu, Yinlin and Fua, Pascal and
                  Salzmann, Mathieu},
  journal =      {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2019
}

@article{shea2024dont,
  title =        {Don't Be So Positive: Negative Step Sizes in Second-Order
                  Methods},
  author =       {Shea, Betty and Schmidt, Mark},
  year =         2024
}

@article{balesni2024towards,
  title =        {Towards evaluations-based safety cases for AI scheming},
  author =       {Balesni, Mikita and Hobbhahn, Marius and Lindner, David and
                  Meinke, Alex and Korbak, Tomek and Clymer, Joshua and
                  Shlegeris, Buck and Scheurer, J{\'e}r{\'e}my and Shah, Rusheb
                  and Goldowsky-Dill, Nicholas and others},
  year =         2024
}

@inproceedings{mcgowan2024efficient,
  title =        {Efficient Model Compression Techniques with FishLeg},
  author =       {McGowan, Jamie and Lai, Wei Sheng and Chen, Weibin and
                  Aldridge, Henry and Clarke, Jools and Garcia, Jezabel R and
                  Xia, Rui and Liang, Yilei and Hennequin, Guillaume and
                  Bernacchia, Alberto},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS),
                  Workshop on Machine Learning and Compression}
}

@misc{lucas2024preserving,
  title =        {Preserving Deep Representations In One-Shot Pruning: A
                  Hessian-Free Second-Order Optimization Framework},
  author =       {Ryan Lucas and Rahul Mazumder},
  year =         2024,
}

@article{enkhbayar2024new,
  title =        {A New Way: Kronecker-Factored Approximate Curvature Deep
                  Hedging and its Benefits},
  author =       {Enkhbayar, Tsogt-Ochir},
  year =         2024
}

@inproceedings{rahma2024training,
  title =        {Training Hamiltonian neural networks without backpropagation},
  author =       {Atamert Rahma and Chinmay Datar and Felix Dietrich},
  year =         2024,
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
}

@article{quinton2024jacobian,
  title =        {Jacobian Descent for Multi-Objective Optimization},
  author =       {Quinton, Pierre and Rey, Val{\'e}rian},
  journal =      {arXiv preprint arXiv:2406.16232},
  year =         2024
}

@inproceedings{dhawan2023efficient,
  title =        {Efficient parametric approximations of neural network function
                  space distance},
  author =       {Dhawan, Nikita and Huang, Sicong and Bae, Juhan and Grosse,
                  Roger Baker},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2023,
}

@inproceedings{yu2024second,
  title =        {Second-order forward-mode optimization of recurrent neural
                  networks for neuroscience},
  author =       {Yu, Youjing and Xia, Rui and Ma, Qingxi and Lengyel,
                  M{\'a}t{\'e} and Hennequin, Guillaume},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2024,
}

@article{epperly2024randomized,
  title =        {Randomized Kaczmarz with tail averaging},
  author =       {Epperly, Ethan N and Goldshlager, Gil and Webber, Robert J},
  year =         2024
}

@inproceedings{shi2024stochastic,
  title =        {Stochastic {T}aylor Derivative Estimator: Efficient amortization
                  for arbitrary differential operators},
  author =       {Zekun Shi and Zheyuan Hu and Min Lin and Kenji Kawaguchi},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2024,
}

@inproceedings{kraemer2024gradients,
  title =        {Gradients of Functions of Large Matrices},
  author =       {Nicholas Kr{\"a}mer and Pablo Moreno-Mu{\~n}oz and Hrittik Roy
                  and S{\o}ren Hauberg},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2024,
}

@inproceedings{potapczynski2024searching,
  title =        {Searching for Efficient Linear Layers over a Continuous Space
                  of Structured Matrices},
  author =       {Andres Potapczynski and Shikai Qiu and Marc Anton Finzi and
                  Christopher Ferri and Zixi Chen and Micah Goldblum and C.
                  Bayan Bruss and Christopher De Sa and Andrew Gordon Wilson},
  booktitle =    {Advances in Neural Information Processing Systems (NeurIPS)},
  year =         2024,
}

@misc{ormaniec2024transformer,
  title =        {What Does It Mean to Be a Transformer? Insights from a
                  Theoretical Hessian Analysis},
  author =       {Weronika Ormaniec and Felix Dangel and Sidak Pal Singh},
  year =         2024,
}

@article{tam2024merging,
  title =        {Merging by Matching Models in Task Parameter Subspaces},
  author =       {Derek Tam and Mohit Bansal and Colin Raffel},
  journal =      {Transactions on Machine Learning Research (TMLR)},
  year =         2024,
}

@article{epperly2024xtrace,
  title =        {Xtrace: Making the most of every sample in stochastic trace
                  estimation},
  author =       {Epperly, Ethan N and Tropp, Joel A and Webber, Robert J},
  journal =      {SIAM Journal on Matrix Analysis and Applications (SIMAX)},
  year =         2024,
}

@article{girard1989montecarlo,
  author =       {Girard, Didier A.},
  journal =      {Numerische Mathematik},
  title =        {A Fast 'Monte-Carlo Cross-Validation' Procedure for Large
                  Least Squares Problems with Noisy Data},
  year =         1989,
}

@article{harris2020array,
  title =        {Array programming with NumPy},
  author =       {Harris, Charles R and Millman, K Jarrod and Van Der Walt,
                  St{\'e}fan J and Gommers, Ralf and Virtanen, Pauli and
                  Cournapeau, David and Wieser, Eric and Taylor, Julian and
                  Berg, Sebastian and Smith, Nathaniel J and others},
  journal =      {Nature},
  year =         2020,
}

@article{virtanen2020scipy,
  title =        {SciPy 1.0: fundamental algorithms for scientific computing in
                  Python},
  author =       {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E and
                  Haberland, Matt and Reddy, Tyler and Cournapeau, David and
                  Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and
                  Bright, Jonathan and others},
  journal =      {Nature methods},
  year =         2020,
}

@inproceedings{rathore2024challenges,
  title =        {Challenges in training PINNs: a loss landscape perspective},
  author =       {Rathore, Pratik and Lei, Weimu and Frangella, Zachary and Lu,
                  Lu and Udell, Madeleine},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2024
}

@book{arbogast1800calcul,
  title =        {Du calcul des d{\'e}rivations},
  author =       {Arbogast, L.F.A.},
  year =         1800,
}

@misc{hardy2006combinatorics,
  title =        {Combinatorics of Partial Derivatives},
  author =       {Michael Hardy},
  year =         2006,
  note =         {arXiv},
}

@article{faa1857note,
  title =        {Note sur une nouvelle formule de calcul diff{\'e}rentiel},
  author =       {Fa{\`a} Di Bruno, Francesco},
  journal =      {Quarterly J. Pure Appl. Math},
  year =         1857
}

@article{griewank_evaluating_1999,
  title =        {Evaluating higher derivative tensors by forward propagation of
                  univariate {Taylor} series},
  volume =       69,
  journal =      {Mathematics of Computation},
  author =       {Griewank, Andreas and Utke, Jean and Walther, Andrea},
  year =         1999,
}

@software{gao2023folx,
  author = {Nicholas Gao and Jonas Köhler and Adam Foster},
  title = {folx - Forward {L}aplacian for {JAX}},
  url = {http://github.com/microsoft/folx},
  year = {2023},
}

@inproceedings{fraenkel1978formulae,
  title =        {Formulae for high derivatives of composite functions},
  author =       {Fraenkel, LE},
  booktitle =    {Mathematical Proceedings of the Cambridge Philosophical
                  Society},
  year =         1978,
}

@article{vahab_physics-informed_2022,
  title =        {A Physics-Informed Neural Network Approach to Solution and
                  Identification of Biharmonic Equations of Elasticity},
  volume =       148,
  number =       2,
  journal =      {Journal of Engineering Mechanics},
  author =       {Vahab, Mohammad and Haghighat, Ehsan and Khaleghi, Maryam and
                  Khalili, Nasser},
  year =         2022
}

@article{vikas_biharm,
  author =       {Dwivedi, Vikas and Srinivasan, Balaji},
  title =        {Solution of Biharmonic Equation in Complicated Geometries With
                  Physics Informed Extreme Learning Machine},
  journal =      {Journal of Computing and Information Science in Engineering},
  volume =       20,
  number =       6,
  year =         2020,
  month =        05,
  issn =         {1530-9827},
}

@article{fa2011solution,
  title =        {Solution of {F}okker-{P}lanck equation for a broad class of drift
                  and diffusion coefficients},
  author =       {Fa, Kwok Sau},
  journal =      {Phys. Rev. E},
  year =         2011,
  publisher =    {American Physical Society},
}

@article{reed2022torch,
  title =        {torch.fx: Practical program capture and transformation for
                  deep learning in python},
  author =       {Reed, James and DeVito, Zachary and He, Horace and Ussery,
                  Ansley and Ansel, Jason},
  journal =      {Proceedings of Machine Learning and Systems (MLSys)},
  year =         2022
}

@inproceedings{wang2025efficiently,
  title =        {Efficiently Access Diffusion Fisher: Within the Outer Product
                  Span Space},
  author =       {Wang, Fangyikang and Yin, Hubery and Zhuang, Shaobin and Zhu,
                  Huminhao and Li, Yinan and Qian, Lei and Zhang, Chao and Zhao,
                  Hanbin and Qian, Hui and Li, Chen},
  booktitle =    {International Conference on Machine Learning (ICML)},
  year =         2025
}