@software{grid2op,
    author = {B. Donnot},
    title = {{Grid2op - A testbed platform to model sequential decision making in power systems. }},
    howpublished = {\url{https://GitHub.com/Grid2Op/grid2op}},
    year = {2020},
    publisher = {GitHub},
}

@misc{sumorl,
    author = {Lucas N. Alegre},
    title = {{SUMO-RL}},
    year = {2019},
    publisher = {GitHub},
    journal = {GitHub repository},
    howpublished = {\url{https://github.com/LucasAlegre/sumo-rl}},
}

@article{lowe2017multi,
  title={Multi-Agent Actor-Critic for Mixed Cooperative-Competitive Environments},
  author={Lowe, Ryan and Wu, Yi and Tamar, Aviv and Harb, Jean and Abbeel, Pieter and Mordatch, Igor},
  journal={Neural Information Processing Systems (NIPS)},
  year={2017}
}

@article{vinyals2017starcraft,
  title={Starcraft {II}: A new challenge for reinforcement learning},
  author={Vinyals, Oriol and Ewalds, Timo and Bartunov, Sergey and Georgiev, Petko and Vezhnevets, Alexander Sasha and Yeo, Michelle and Makhzani, Alireza and K{\"u}ttler, Heinrich and Agapiou, John and Schrittwieser, Julian and others},
  journal={arXiv preprint arXiv:1708.04782},
  year={2017}
}

@article{jin2024approximate,
  title={Approximate Global Convergence of Independent Learning in Multi-Agent Systems},
  author={Jin, Ruiyang and Chen, Zaiwei and Lin, Yiheng and Song, Jie and Wierman, Adam},
  journal={arXiv preprint arXiv:2405.19811},
  year={2024}
}

@inproceedings{monroc2024wind,
  title={Wind farm control with cooperative multi-agent reinforcement learning},
  author={Monroc, Claire Bizon and Bu{\v{s}}i{\'c}, Ana and Dubuc, Donatien and Zhu, Jiamin},
  booktitle={ICML 2024 Workshop: Aligning Reinforcement Learning Experimentalists and Theorists (ARLET 2024)},
  year={2024}
}

@article{qu2022scalable,
  title={Scalable reinforcement learning for multiagent networked systems},
  author={Qu, Guannan and Wierman, Adam and Li, Na},
  journal={Operations Research},
  volume={70},
  number={6},
  pages={3601--3628},
  year={2022},
  publisher={INFORMS}
}

@article{varaiya2013max,
  title={Max pressure control of a network of signalized intersections},
  author={Varaiya, Pravin},
  journal={Transportation Research Part C: Emerging Technologies},
  volume={36},
  pages={177--195},
  year={2013},
  publisher={Elsevier}
}

@article{chu2020multi,
  title={Multi-agent reinforcement learning for networked system control},
  author={Chu, Tianshu and Chinchali, Sandeep and Katti, Sachin},
  journal={arXiv preprint arXiv:2004.01339},
  year={2020}
}

@article{agarwal2024multi,
  title={Multi-Armed Bandits with Network Interference},
  author={Agarwal, Abhineet and Agarwal, Anish and Masoero, Lorenzo and Whitehouse, Justin},
  journal={arXiv preprint arXiv:2405.18621},
  year={2024}
}

@article{ou2022networked,
  title={Networked restless multi-armed bandits for mobile interventions},
  author={Ou, Han-Ching and Siebenbrunner, Christoph and Killian, Jackson and Brooks, Meredith B and Kempe, David and Vorobeychik, Yevgeniy and Tambe, Milind},
  journal={arXiv preprint arXiv:2201.12408},
  year={2022}
}

@article{hu2024decentralized,
  title={Decentralized graph-based multi-agent reinforcement learning using reward machines},
  author={Hu, Jueming and Xu, Zhe and Wang, Weichang and Qu, Guannan and Pang, Yutian and Liu, Yongming},
  journal={Neurocomputing},
  volume={564},
  pages={126974},
  year={2024},
  publisher={Elsevier}
}

@article{lehna_managing_2023,
	title = {Managing power grids through topology actions: A comparative study between advanced rule-based and reinforcement learning agents},
	issn = {2666-5468},
	url = {https://www.sciencedirect.com/science/article/pii/S2666546823000484},
	doi = {https://doi.org/10.1016/j.egyai.2023.100276},
	pages = {100276},
	journaltitle = {Energy and {AI}},
	author = {Lehna, Malte and Viebahn, Jan and Marot, Antoine and Tomforde, Sven and Scholz, Christoph},
	date = {2023},
}

@inproceedings{guestrin2001max,
  title={Max-norm projections for factored MDPs},
  author={Guestrin, Carlos and Koller, Daphne and Parr, Ronald},
  booktitle={IJCAI},
  volume={1},
  pages={673--682},
  year={2001}
}

@inproceedings{guestrin2002coordinated,
  title={Coordinated reinforcement learning},
  author={Guestrin, Carlos and Lagoudakis, Michail and Parr, Ronald},
  booktitle={ICML},
  volume={2},
  pages={227--234},
  year={2002},
  organization={Citeseer}
}

@article{holley1975ergodic,
  title={Ergodic theorems for weakly interacting infinite systems and the voter model},
  author={Holley, Richard A and Liggett, Thomas M},
  journal={The annals of probability},
  pages={643--663},
  year={1975},
  publisher={JSTOR}
}

@article{terry2021pettingzoo,
  title={Pettingzoo: Gym for multi-agent reinforcement learning},
  author={Terry, J and Black, Benjamin and Grammel, Nathaniel and Jayakumar, Mario and Hari, Ananth and Sullivan, Ryan and Santos, Luis S and Dieffendahl, Clemens and Horsch, Caroline and Perez-Vicente, Rodrigo and others},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={15032--15043},
  year={2021}
}

@article{towers2024gymnasium,
  title={Gymnasium: A Standard Interface for Reinforcement Learning Environments},
  author={Towers, Mark and Kwiatkowski, Ariel and Terry, Jordan and Balis, John U and De Cola, Gianluca and Deleu, Tristan and Goul{\~a}o, Manuel and Kallinteris, Andreas and Krimmel, Markus and KG, Arjun and others},
  journal={arXiv preprint arXiv:2407.17032},
  year={2024}
}

@book{oliehoek2016concise,
  title={A concise introduction to decentralized POMDPs},
  author={Oliehoek, Frans A and Amato, Christopher and others},
  volume={1},
  year={2016},
  publisher={Springer}
}

@article{de2020independent,
  title={Is independent learning all you need in the starcraft multi-agent challenge?},
  author={De Witt, Christian Schroeder and Gupta, Tarun and Makoviichuk, Denys and Makoviychuk, Viktor and Torr, Philip HS and Sun, Mingfei and Whiteson, Shimon},
  journal={arXiv preprint arXiv:2011.09533},
  year={2020}
}

@book{puterman2014markov,
  title={Markov decision processes: discrete stochastic dynamic programming},
  author={Puterman, Martin L},
  year={2014},
  publisher={John Wiley \& Sons}
}

@article{kaelbling1998planning,
  title={Planning and acting in partially observable stochastic domains},
  author={Kaelbling, Leslie Pack and Littman, Michael L and Cassandra, Anthony R},
  journal={Artificial intelligence},
  volume={101},
  number={1-2},
  pages={99--134},
  year={1998},
  publisher={Elsevier}
}

@inproceedings{nair2005networked,
  title={Networked distributed POMDPs: A synthesis of distributed constraint optimization and POMDPs},
  author={Nair, Ranjit and Varakantham, Pradeep and Tambe, Milind and Yokoo, Makoto},
  booktitle={AAAI},
  volume={5},
  pages={133--139},
  year={2005}
}

@article{marot2020learning,
  title={Learning to run a power network challenge for training topology controllers},
  author={Marot, Antoine and Donnot, Benjamin and Romero, Camilo and Donon, Balthazar and Lerousseau, Marvin and Veyrin-Forrer, Luca and Guyon, Isabelle},
  journal={Electric Power Systems Research},
  volume={189},
  pages={106635},
  year={2020},
  publisher={Elsevier}
}

@article{bizon2024wfcrl,
  title={{WFCRL}: A Multi-Agent Reinforcement Learning Benchmark for Wind Farm Control},
  author={Bizon Monroc, Claire and Busic, Ana and Dubuc, Donatien and Zhu, Jiamin},
  journal={Advances in Neural Information Processing Systems},
  volume={37},
  pages={133254--133281},
  year={2024}
}

@inproceedings{SUMO2018,
          title = {Microscopic Traffic Simulation using {SUMO}},
         author = {Pablo Alvarez Lopez and Michael Behrisch and Laura Bieker-Walz and Jakob Erdmann and Yun-Pang Fl{\"o}tter{\"o}d and Robert Hilbrich and Leonhard L{\"u}cken and Johannes Rummel and Peter Wagner and Evamarie Wie{\ss}ner},
      publisher = {IEEE},
      booktitle = {The 21st IEEE International Conference on Intelligent Transportation Systems},
           year = {2018},
        journal = {IEEE Intelligent Transportation Systems Conference (ITSC)},
       keywords = {traffic simulation, modelling, optimization},
            url = {https://elib.dlr.de/124092/}
 }

@article{bettini2022vmas,
  title = {VMAS: A Vectorized Multi-Agent Simulator for Collective Robot Learning},
  author = {Bettini, Matteo and Kortvelesy, Ryan and Blumenkamp, Jan and Prorok, Amanda},
  year = {2022},
  journal={The 16th International Symposium on Distributed Autonomous Robotic Systems},
  publisher={Springer}
}

@article{terry2020arcade,
  Title = {Multiplayer Support for the Arcade Learning Environment},
  Author = {Terry, J K and Black, Benjamin},
  journal={arXiv preprint arXiv:2009.09341},
  year={2020}
}

@inproceedings{gupta2017cooperative,
  title={Cooperative multi-agent control using deep reinforcement learning},
  author={Gupta, Jayesh K and Egorov, Maxim and Kochenderfer, Mykel},
  booktitle={International Conference on Autonomous Agents and Multiagent Systems},
  pages={66--83},
  year={2017},
  organization={Springer}
}

@article{chen2022reinforcement,
  title={Reinforcement learning for selective key applications in power systems: Recent advances and future challenges},
  author={Chen, Xin and Qu, Guannan and Tang, Yujie and Low, Steven and Li, Na},
  journal={IEEE Transactions on Smart Grid},
  volume={13},
  number={4},
  pages={2935--2958},
  year={2022},
  publisher={IEEE}
}

@article{qi2021federated,
  title={Federated reinforcement learning: techniques, applications, and open challenges},
  author={Qi, Jiaju and Zhou, Qihao and Lei, Lei and Zheng, Kan},
  journal={Intelligence \& Robotics},
  volume={1},
  number={1},
  pages={18--57},
  year={2021},
  publisher={OAE Publishing Inc.}
}

@article{huang2022cleanrl,
  author  = {Shengyi Huang and Rousslan Fernand Julien Dossa and Chang Ye and Jeff Braga and Dipam Chakraborty and Kinal Mehta and João G.M. Araújo},
  title   = {CleanRL: High-quality Single-file Implementations of Deep Reinforcement Learning Algorithms},
  journal = {Journal of Machine Learning Research},
  year    = {2022},
  volume  = {23},
  number  = {274},
  pages   = {1--18},
  url     = {http://jmlr.org/papers/v23/21-1342.html}
}

@article{zhang2021survey,
  title={A survey on federated learning},
  author={Zhang, Chen and Xie, Yu and Bai, Hang and Yu, Bin and Li, Weihong and Gao, Yuan},
  journal={Knowledge-Based Systems},
  volume={216},
  pages={106775},
  year={2021},
  publisher={Elsevier}
}
@book{barto2021reinforcement,
  title={Reinforcement learning: An introduction},
  author={Sutton, Richard S and Barto, Andrew G and others},
  volume={1},
  number={1},
  year={1998},
  publisher={MIT press Cambridge}
}

@inproceedings{tan1993multi,
  title={Multi-agent reinforcement learning: Independent vs. cooperative agents},
  author={Tan, Ming},
  booktitle={Proceedings of the tenth international conference on machine learning},
  pages={330--337},
  year={1993}
}

@article{haider2024networkgym,
  title={NetworkGym: Reinforcement Learning Environments for Multi-Access Traffic Management in Network Simulation},
  author={Haider, Momin and Yin, Ming and Zhang, Menglei and Gupta, Arpit and Zhu, Jing and Wang, Yu-Xiang},
  journal={Proceedings of Machine Learning Research},
  year={2024},
  publisher={NeurIPS 2024 Dataset and Benchmark}
}

@misc{skrynnik2024pogema,
      title={POGEMA: A Benchmark Platform for Cooperative Multi-Agent Navigation}, 
      author={Alexey Skrynnik and Anton Andreychuk and Anatolii Borzilov and Alexander Chernyavskiy and Konstantin Yakovlev and Aleksandr Panov},
      year={2024},
      eprint={2407.14931},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2407.14931}, 
}

@article{bellman1966dynamic,
  title={Dynamic programming},
  author={Bellman, Richard},
  journal={science},
  volume={153},
  number={3731},
  pages={34--37},
  year={1966},
  publisher={American Association for the Advancement of Science}
}

@article{yu2022surprising,
  title={The surprising effectiveness of ppo in cooperative multi-agent games},
  author={Yu, Chao and Velu, Akash and Vinitsky, Eugene and Gao, Jiaxuan and Wang, Yu and Bayen, Alexandre and Wu, Yi},
  journal={Advances in neural information processing systems},
  volume={35},
  pages={24611--24624},
  year={2022}
}

@book{ahuja1993network,
  title={Network flows: theory, algorithms, and applications},
  author={Ahuja, Ravindra K and Magnanti, Thomas L and Orlin, James B and others},
  volume={1},
  year={1993},
  publisher={Prentice hall Englewood Cliffs, NJ}
}

@inproceedings{oliehoek2008exploiting,
  title={Exploiting locality of interaction in factored Dec-POMDPs},
  author={Oliehoek, Frans A and Spaan, Matthijs TJ and Vlassis, Nikos and Whiteson, Shimon},
  booktitle={Int. Joint Conf. on Autonomous Agents and Multi-Agent Systems},
  year={2008}
}

@inproceedings{yeh2023sustaingym,
    title = {{SustainGym}: Reinforcement Learning Environments for Sustainable Energy Systems},
    author = {Yeh, Christopher and Li, Victor and Datta, Rajeev and Arroyo, Julio and Zhang, Chi and Chen, Yize and Hosseini, Mehdi and Golmohammadi, Azarang and Shi, Yuanyuan and Yue, Yisong and Wierman, Adam},
    year = 2023,
    month = 12,
    booktitle = {Thirty-seventh Conference on Neural Information Processing Systems Datasets and Benchmarks Track},
    address = {New Orleans, LA, USA},
    url = {https://openreview.net/forum?id=vZ9tA3o3hr}
}

@article{van2023multi,
  title={Multi-agent reinforcement learning for power grid topology optimization},
  author={van der Sar, Erica and Zocca, Alessandro and Bhulai, Sandjai},
  journal={arXiv preprint arXiv:2310.02605},
  year={2023}
}

@article{qu2020scalable,
  title={Scalable multi-agent reinforcement learning for networked systems with average reward},
  author={Qu, Guannan and Lin, Yiheng and Wierman, Adam and Li, Na},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={2074--2086},
  year={2020}
}

@inproceedings{soton351021,
       booktitle = {Proceedings of the 23rd International Joint Conference on AI (IJCAI) (03/08/13 - 09/08/13)},
           title = {Monte-Carlo expectation maximization for decentralized POMDPs},
          author = {Feng Wu and S Zilberstein and N.R. Jennings},
            year = {2013},
           pages = {397--403},
             url = {https://eprints.soton.ac.uk/351021/},
        abstract = {We address two significant drawbacks of state-of-the-art solvers of decentralized POMDPs (DECPOMDPs): the reliance on complete knowledge of the model and limited scalability as the complexity of the domain grows. We extend a recently proposed approach for solving DEC-POMDPs via a reduction to the maximum likelihood problem, which in turn can be solved using EM. We introduce a model-free version of this approach that employs Monte-Carlo EM (MCEM). While a na{\"i}ve implementation of MCEM is inadequate in multiagent settings, we introduce several improvements in sampling that produce high-quality results on a variety of DEC-POMDP benchmarks, including large problems with thousands of agents.}
}

@inproceedings{ooi1996decentralized,
  title={Decentralized control of a multiple access broadcast channel: Performance bounds},
  author={Ooi, James M and Wornell, Gregory W},
  booktitle={Proceedings of 35th IEEE Conference on Decision and Control},
  volume={1},
  pages={293--298},
  year={1996},
  organization={IEEE}
}

@article{mahajan2016decentralized,
  title={Decentralized stochastic control},
  author={Mahajan, Aditya and Mannan, Mehnaz},
  journal={Annals of Operations Research},
  volume={241},
  number={1},
  pages={109--126},
  year={2016},
  publisher={Springer}
}

@inproceedings{bargiacchi2021cooperative,
  title={Cooperative Prioritized Sweeping.},
  author={Bargiacchi, Eugenio and Verstraeten, Timothy and Roijers, Diederik M},
  year={2021},
  booktitle = {Proceedings of the 20th International Conference on Autonomous Agents and MultiAgent Systems (AAMAS)},
  pages     = {160--168}
}

@inproceedings{bianchi2024developing,
  title={Developing safe and explainable autonomous agents: from simulation to the real world},
  author={Bianchi, Federico and Castellini, Alberto and Farinelli, Alessandro and Marzari, Luca and Meli, Daniele and Trotti, Francesco and Veronese, Celeste and others},
  booktitle={CEUR WORKSHOP PROCEEDINGS},
  pages={89--94},
  year={2024}
}

@inproceedings{zhang2018fully,
  title={Fully decentralized multi-agent reinforcement learning with networked agents},
  author={Zhang, Kaiqing and Yang, Zhuoran and Liu, Han and Zhang, Tong and Basar, Tamer},
  booktitle={International conference on machine learning},
  pages={5872--5881},
  year={2018},
  organization={PMLR}
}

@ARTICLE{gnnsurvey2023,
  author={Munikoti, Sai and Agarwal, Deepesh and Das, Laya and Halappanavar, Mahantesh and Natarajan, Balasubramaniam},
  journal={IEEE Transactions on Neural Networks and Learning Systems}, 
  title={Challenges and Opportunities in Deep Reinforcement Learning With Graph Neural Networks: A Comprehensive Review of Algorithms and Applications}, 
  year={2024},
  volume={35},
  number={11},
  pages={15051-15071},
  keywords={Graph neural networks;Deep learning;Markov processes;Surveys;Mathematical models;Computer architecture;Computational modeling;Deep learning;deep reinforcement learning (DRL);graph neural network (GNN);hybrid DRL-GNN;survey},
  doi={10.1109/TNNLS.2023.3283523}}

@ARTICLE{scalability2024,
  author={Liu, Dingbang and Ren, Fenghui and Yan, Jun and Su, Guoxin and Gu, Wen and Kato, Shohei},
  journal={IEEE Access}, 
  title={Scaling Up Multi-Agent Reinforcement Learning: An Extensive Survey on Scalability Issues}, 
  year={2024},
  volume={12},
  number={},
  pages={94610-94631},
  keywords={Scalability;Complexity theory;Robustness;Surveys;Deep reinforcement learning;Task analysis;Games;Multi-agent systems;Collective intelligence;Multi-agent learning;reinforcement learning;scalability;collective learning},
  doi={10.1109/ACCESS.2024.3410318}}

@article{rashid2020monotonic,
  title={Monotonic value function factorisation for deep multi-agent reinforcement learning},
  author={Rashid, Tabish and Samvelyan, Mikayel and De Witt, Christian Schroeder and Farquhar, Gregory and Foerster, Jakob and Whiteson, Shimon},
  journal={Journal of Machine Learning Research},
  volume={21},
  number={178},
  pages={1--51},
  year={2020}
}

@article{sunehag2017value,
  title={Value-decomposition networks for cooperative multi-agent learning},
  author={Sunehag, Peter and Lever, Guy and Gruslys, Audrunas and Czarnecki, Wojciech Marian and Zambaldi, Vinicius and Jaderberg, Max and Lanctot, Marc and Sonnerat, Nicolas and Leibo, Joel Z and Tuyls, Karl and others},
  journal={arXiv preprint arXiv:1706.05296},
  year={2017}
}