@misc{kusner2018cf,
      title={Counterfactual {F}airness}, 
      author={Matt J. Kusner and Joshua R. Loftus and Chris Russell and Ricardo Silva},
      year={2018},
      eprint={1703.06856},
      archivePrefix={arXiv},
      primaryClass={stat.ML}
}

@misc{wang2025cfrl,
      title={Counterfactually Fair Reinforcement Learning via Sequential Data Preprocessing}, 
      author={Jitao Wang and Chengchun Shi and John D. Piette and Joshua R. Loftus and Donglin Zeng and Zhenke Wu},
      year={2025},
      eprint={2501.06366},
      archivePrefix={arXiv},
      primaryClass={stat.ML}
}

@misc{weerts2023fairlearn,
      title={Fairlearn: Assessing and Improving Fairness of AI Systems},
      author={Hilde Weerts and Miroslav Dudík and Richard Edgar and Adrin Jalali and Roman Lutz and Michael Madaio},
      journal={Journal of Machine Learning Research},
      year={2023},
      volume={24},
      number={257},
      pages={1--8}
}

@misc{aif360-oct-2018,
    title = "{AI Fairness} 360:  An Extensible Toolkit for Detecting, Understanding, and Mitigating Unwanted Algorithmic Bias",
    author = {Rachel K. E. Bellamy and Kuntal Dey and Michael Hind and
	Samuel C. Hoffman and Stephanie Houde and Kalapriya Kannan and
	Pranay Lohia and Jacquelyn Martino and Sameep Mehta and
	Aleksandra Mojsilovic and Seema Nagar and Karthikeyan Natesan Ramamurthy and
	John Richards and Diptikalyan Saha and Prasanna Sattigeri and
	Moninder Singh and Kush R. Varshney and Yunfeng Zhang},
    month = oct,
    year = {2018}
}

@inproceedings{fairness_gym,
author = {D'Amour, Alexander and Srinivasan, Hansa and Atwood, James and Baljekar, Pallavi and Sculley, D. and Halpern, Yoni},
title = {Fairness is Not Static: Deeper Understanding of Long Term Fairness via Simulation Studies},
year = {2020},
isbn = {9781450369367},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
doi = {10.1145/3351095.3372878},
booktitle = {Proceedings of the 2020 Conference on Fairness, Accountability, and Transparency},
pages = {525-534},
numpages = {10},
location = {Barcelona, Spain},
series = {FAccT '20}
}

@article{d3rlpy,
  author  = {Takuma Seno and Michita Imai},
  title   = {d3rlpy: An Offline Deep Reinforcement Learning Library},
  journal = {Journal of Machine Learning Research},
  year    = {2022},
  volume  = {23},
  number  = {315},
  pages   = {1--20}
}

@article{towers2024gymnasium,
  title={Gymnasium: A Standard Interface for Reinforcement Learning Environments},
  author={Towers, Mark and Kwiatkowski, Ariel and Terry, Jordan and Balis, John U and De Cola, Gianluca and Deleu, Tristan and Goul{\~a}o, Manuel and Kallinteris, Andreas and Krimmel, Markus and KG, Arjun and others},
  journal={arXiv preprint arXiv:2407.17032},
  year={2024}
}


@Article{piette2023powerED,
author="Piette, John D
and Thomas, Laura
and Newman, Sean
and Marinec, Nicolle
and Krauss, Joel
and Chen, Jenny
and Wu, Zhenke
and Bohnert, Amy S B",
title="An Automatically Adaptive Digital Health Intervention to Decrease Opioid-Related Risk While Conserving Counselor Time: Quantitative Analysis of Treatment Decisions Based on Artificial Intelligence and Patient-Reported Risk Measures",
journal="Journal of Medical Internet Research",
year="2023",
month="Jul",
day="11",
volume="25",
pages="e44165",
keywords="artificial intelligence; opioid safety; telehealth; reinforcement learning; pain management",
abstract="Background: Some patients prescribed opioid analgesic (OA) medications for pain experience serious side effects, including dependence, sedation, and overdose. As most patients are at low risk for OA-related harms, risk reduction interventions requiring multiple counseling sessions are impractical on a large scale. Objective: This study evaluates whether an intervention based on reinforcement learning (RL), a field of artificial intelligence, learned through experience to personalize interactions with patients with pain discharged from the emergency department (ED) and decreased self-reported OA misuse behaviors while conserving counselors' time. Methods: We used data representing 2439 weekly interactions between a digital health intervention (``Prescription Opioid Wellness and Engagement Research in the ED'' [PowerED]) and 228 patients with pain discharged from 2 EDs who reported recent opioid misuse. During each patient's 12 weeks of intervention, PowerED used RL to select from 3 treatment options: a brief motivational message delivered via an interactive voice response (IVR) call, a longer motivational IVR call, or a live call from a counselor. The algorithm selected session types for each patient each week, with the goal of minimizing OA risk, defined in terms of a dynamic score reflecting patient reports during IVR monitoring calls. When a live counseling call was predicted to have a similar impact on future risk as an IVR message, the algorithm favored IVR to conserve counselor time. We used logit models to estimate changes in the relative frequency of each session type as PowerED gained experience. Poisson regression was used to examine the changes in self-reported OA risk scores over calendar time, controlling for the ordinal session number (1st to 12th). Results: Participants on average were 40 (SD 12.7) years of age; 66.7{\%} (152/228) were women and 51.3{\%} (117/228) were unemployed. Most participants (175/228, 76.8{\%}) reported chronic pain, and 46.2{\%} (104/225) had moderate to severe depressive symptoms. As PowerED gained experience through interactions over a period of 142 weeks, it delivered fewer live counseling sessions than brief IVR sessions (P=.006) and extended IVR sessions (P<.001). Live counseling sessions were selected 33.5{\%} of the time in the first 5 weeks of interactions (95{\%} CI 27.4{\%}-39.7{\%}) but only for 16.4{\%} of sessions (95{\%} CI 12.7{\%}-20{\%}) after 125 weeks. Controlling for each patient's changes during the course of treatment, this adaptation of treatment-type allocation led to progressively greater improvements in self-reported OA risk scores (P<.001) over calendar time, as measured by the number of weeks since enrollment began. Improvement in risk behaviors over time was especially pronounced among patients with the highest risk at baseline (P=.02). Conclusions: The RL-supported program learned which treatment modalities worked best to improve self-reported OA risk behaviors while conserving counselors' time. RL-supported interventions represent a scalable solution for patients with pain receiving OA prescriptions. Trial Registration: Clinicaltrials.gov NCT02990377; https://classic.clinicaltrials.gov/ct2/show/NCT02990377 ",
issn="1438-8871",
doi="10.2196/44165"
}

@InProceedings{riedmiller2005fqi,
author="Riedmiller, Martin",
editor="Gama, Jo{\~a}o
and Camacho, Rui
and Brazdil, Pavel B.
and Jorge, Al{\'i}pio M{\'a}rio
and Torgo, Lu{\'i}s",
title="Neural Fitted {Q} Iteration -- First Experiences with a Data Efficient Neural Reinforcement Learning Method",
booktitle="Machine Learning: ECML 2005",
year="2005",
publisher="Springer Berlin Heidelberg",
address="Berlin, Heidelberg",
pages="317--328",
abstract="This paper introduces NFQ, an algorithm for efficient and effective training of a Q-value function represented by a multi-layer perceptron. Based on the principle of storing and reusing transition experiences, a model-free, neural network based Reinforcement Learning algorithm is proposed. The method is evaluated on three benchmark problems. It is shown empirically, that reasonably few interactions with the plant are needed to generate control policies of high quality.",
isbn="978-3-540-31692-3"
}


