@article{castro_polynomial_2009,
  title = {Polynomial Calculation of the {{Shapley}} Value Based on Sampling},
  author = {Castro, Javier and G{\'o}mez, Daniel and Tejada, Juan},
  year = {2009},
  month = may,
  journal = {Computers \& Operations Research},
  series = {Selected Papers Presented at the {{Tenth International Symposium}} on {{Locational Decisions}} ({{ISOLDE X}})},
  volume = {36},
  number = {5},
  pages = {1726--1730},
  issn = {0305-0548},
  doi = {10.1016/j.cor.2008.04.004},
  url = {http://www.sciencedirect.com/science/article/pii/S0305054808000804},
  urldate = {2020-11-21},
  abstract = {In this paper we develop a polynomial method based on sampling theory that can be used to estimate the Shapley value (or any semivalue) for cooperative games. Besides analyzing the complexity problem, we examine some desirable statistical properties of the proposed approach and provide some computational results.},
  langid = {english}
}

@inproceedings{ghorbani_data_2019,
  title = {Data {{Shapley}}: {{Equitable Valuation}} of {{Data}} for {{Machine Learning}}},
  shorttitle = {Data {{Shapley}}},
  booktitle = {Proceedings of the 36th {{International Conference}} on {{Machine Learning}}, {{PMLR}}},
  author = {Ghorbani, Amirata and Zou, James},
  year = {2019},
  month = may,
  eprint = {1904.02868},
  pages = {2242--2251},
  publisher = {{PMLR}},
  issn = {2640-3498},
  url = {http://proceedings.mlr.press/v97/ghorbani19c.html},
  urldate = {2020-11-01},
  abstract = {As data becomes the fuel driving technological and economic growth, a fundamental challenge is how to quantify the value of data in algorithmic predictions and decisions. For example, in healthcare and consumer markets, it has been suggested that individuals should be compensated for the data that they generate, but it is not clear what is an equitable valuation for individual data. In this work, we develop a principled framework to address data valuation in the context of supervised machine learning. Given a learning algorithm trained on n data points to produce a predictor, we propose data Shapley as a metric to quantify the value of each training datum to the predictor performance. Data Shapley uniquely satisfies several natural properties of equitable data valuation. We develop Monte Carlo and gradient-based methods to efficiently estimate data Shapley values in practical settings where complex learning algorithms, including neural networks, are trained on large datasets. In addition to being equitable, extensive experiments across biomedical, image and synthetic data demonstrate that data Shapley has several other benefits: 1) it is more powerful than the popular leave-one-out or leverage score in providing insight on what data is more valuable for a given learning task; 2) low Shapley value data effectively capture outliers and corruptions; 3) high Shapley value data inform what type of new data to acquire to improve the predictor.},
  archiveprefix = {arxiv},
  langid = {english},
  keywords = {notion}
}

@inproceedings{jia_efficient_2019,
  title = {Towards {{Efficient Data Valuation Based}} on the {{Shapley Value}}},
  booktitle = {Proceedings of the 22nd {{International Conference}} on {{Artificial Intelligence}} and {{Statistics}}},
  author = {Jia, Ruoxi and Dao, David and Wang, Boxin and Hubis, Frances Ann and Hynes, Nick and G{\"u}rel, Nezihe Merve and Li, Bo and Zhang, Ce and Song, Dawn and Spanos, Costas J.},
  year = {2019},
  month = apr,
  pages = {1167--1176},
  publisher = {{PMLR}},
  issn = {2640-3498},
  url = {http://proceedings.mlr.press/v89/jia19a.html},
  urldate = {2021-02-12},
  abstract = {``How much is my data worth?'' is an increasingly common question posed by organizations and individuals alike. An answer to this question could allow, for instance, fairly distributing profits...},
  langid = {english},
  keywords = {notion}
}

@article{jia_efficient_2019a,
  title = {Efficient Task-Specific Data Valuation for Nearest Neighbor Algorithms},
  shorttitle = {{{VLDB}} 2019},
  author = {Jia, Ruoxi and Dao, David and Wang, Boxin and Hubis, Frances Ann and Gurel, Nezihe Merve and Li, Bo and Zhang, Ce and Spanos, Costas and Song, Dawn},
  year = {2019},
  month = jul,
  journal = {Proceedings of the VLDB Endowment},
  volume = {12},
  number = {11},
  pages = {1610--1623},
  issn = {2150-8097},
  doi = {10.14778/3342263.3342637},
  url = {https://doi.org/10.14778/3342263.3342637},
  urldate = {2021-02-12},
  abstract = {Given a data set D containing millions of data points and a data consumer who is willing to pay for \$X to train a machine learning (ML) model over D, how should we distribute this \$X to each data point to reflect its "value"? In this paper, we define the "relative value of data" via the Shapley value, as it uniquely possesses properties with appealing real-world interpretations, such as fairness, rationality and decentralizability. For general, bounded utility functions, the Shapley value is known to be challenging to compute: to get Shapley values for all N data points, it requires O(2N) model evaluations for exact computation and O(N log N) for ({$\epsilon$}, {$\delta$})-approximation. In this paper, we focus on one popular family of ML models relying on K-nearest neighbors (KNN). The most surprising result is that for unweighted KNN classifiers and regressors, the Shapley value of all N data points can be computed, exactly, in O(N log N) time - an exponential improvement on computational complexity! Moreover, for ({$\epsilon$}, {$\delta$})-approximation, we are able to develop an algorithm based on Locality Sensitive Hashing (LSH) with only sublinear complexity O(Nh({$\epsilon$}, K) log N) when {$\epsilon$} is not too small and K is not too large. We empirically evaluate our algorithms on up to 10 million data points and even our exact algorithm is up to three orders of magnitude faster than the baseline approximation algorithm. The LSH-based approximation algorithm can accelerate the value calculation process even further. We then extend our algorithm to other scenarios such as (1) weighed KNN classifiers, (2) different data points are clustered by different data curators, and (3) there are data analysts providing computation who also requires proper valuation. Some of these extensions, although also being improved exponentially, are less practical for exact computation (e.g., O(NK) complexity for weigthed KNN). We thus propose an Monte Carlo approximation algorithm, which is O(N(log N)2/(log K)2) times more efficient than the baseline approximation algorithm.},
  langid = {english},
  keywords = {notion}
}

@inproceedings{koh_understanding_2017,
  title = {Understanding {{Black-box Predictions}} via {{Influence Functions}}},
  booktitle = {Proceedings of the 34th {{International Conference}} on {{Machine Learning}}},
  author = {Koh, Pang Wei and Liang, Percy},
  year = {2017},
  month = jul,
  eprint = {1703.04730},
  pages = {1885--1894},
  publisher = {{PMLR}},
  url = {https://proceedings.mlr.press/v70/koh17a.html},
  urldate = {2022-05-09},
  abstract = {How can we explain the predictions of a black-box model? In this paper, we use influence functions \textemdash{} a classic technique from robust statistics \textemdash{} to trace a model's prediction through the learning algorithm and back to its training data, thereby identifying training points most responsible for a given prediction. To scale up influence functions to modern machine learning settings, we develop a simple, efficient implementation that requires only oracle access to gradients and Hessian-vector products. We show that even on non-convex and non-differentiable models where the theory breaks down, approximations to influence functions can still provide valuable information. On linear models and convolutional neural networks, we demonstrate that influence functions are useful for multiple purposes: understanding model behavior, debugging models, detecting dataset errors, and even creating visually-indistinguishable training-set attacks.},
  archiveprefix = {arxiv},
  langid = {english},
  keywords = {notion}
}

@inproceedings{kwon_beta_2022,
  title = {Beta {{Shapley}}: A {{Unified}} and {{Noise-reduced Data Valuation Framework}} for {{Machine Learning}}},
  shorttitle = {Beta {{Shapley}}},
  booktitle = {Proceedings of the 25th {{International Conference}} on {{Artificial Intelligence}} and {{Statistics}} ({{AISTATS}}) 2022,},
  author = {Kwon, Yongchan and Zou, James},
  year = {2022},
  month = jan,
  volume = {151},
  eprint = {2110.14049},
  publisher = {{PMLR}},
  address = {{Valencia, Spain}},
  url = {http://arxiv.org/abs/2110.14049},
  urldate = {2022-04-06},
  abstract = {Data Shapley has recently been proposed as a principled framework to quantify the contribution of individual datum in machine learning. It can effectively identify helpful or harmful data points for a learning algorithm. In this paper, we propose Beta Shapley, which is a substantial generalization of Data Shapley. Beta Shapley arises naturally by relaxing the efficiency axiom of the Shapley value, which is not critical for machine learning settings. Beta Shapley unifies several popular data valuation methods and includes data Shapley as a special case. Moreover, we prove that Beta Shapley has several desirable statistical properties and propose efficient algorithms to estimate it. We demonstrate that Beta Shapley outperforms state-of-the-art data valuation methods on several downstream ML tasks such as: 1) detecting mislabeled training data; 2) learning with subsamples; and 3) identifying points whose addition or removal have the largest positive or negative impact on the model.},
  archiveprefix = {arxiv},
  langid = {english},
  keywords = {notion}
}

@inproceedings{okhrati_multilinear_2021,
  title = {A {{Multilinear Sampling Algorithm}} to {{Estimate Shapley Values}}},
  booktitle = {2020 25th {{International Conference}} on {{Pattern Recognition}} ({{ICPR}})},
  author = {Okhrati, Ramin and Lipani, Aldo},
  year = {2021},
  month = jan,
  eprint = {2010.12082},
  pages = {7992--7999},
  publisher = {{IEEE}},
  issn = {1051-4651},
  doi = {10.1109/ICPR48806.2021.9412511},
  url = {https://ieeexplore.ieee.org/abstract/document/9412511},
  abstract = {Shapley values are great analytical tools in game theory to measure the importance of a player in a game. Due to their axiomatic and desirable properties such as efficiency, they have become popular for feature importance analysis in data science and machine learning. However, the time complexity to compute Shapley values based on the original formula is exponential, and as the number of features increases, this becomes infeasible. Castro et al. [1] developed a sampling algorithm, to estimate Shapley values. In this work, we propose a new sampling method based on a multilinear extension technique as applied in game theory. The aim is to provide a more efficient (sampling) method for estimating Shapley values. Our method is applicable to any machine learning model, in particular for either multiclass classifications or regression problems. We apply the method to estimate Shapley values for multilayer perceptrons (MLPs) and through experimentation on two datasets, we demonstrate that our method provides more accurate estimations of the Shapley values by reducing the variance of the sampling statistics.},
  archiveprefix = {arxiv},
  langid = {english},
  keywords = {notion}
}

@misc{schioppa_scaling_2021,
  title = {Scaling {{Up Influence Functions}}},
  author = {Schioppa, Andrea and Zablotskaia, Polina and Vilar, David and Sokolov, Artem},
  year = {2021},
  month = dec,
  number = {arXiv:2112.03052},
  eprint = {arXiv:2112.03052},
  publisher = {{arXiv}},
  doi = {10.48550/arXiv.2112.03052},
  url = {http://arxiv.org/abs/2112.03052},
  urldate = {2023-03-10},
  abstract = {We address efficient calculation of influence functions for tracking predictions back to the training data. We propose and analyze a new approach to speeding up the inverse Hessian calculation based on Arnoldi iteration. With this improvement, we achieve, to the best of our knowledge, the first successful implementation of influence functions that scales to full-size (language and vision) Transformer models with several hundreds of millions of parameters. We evaluate our approach on image classification and sequence-to-sequence tasks with tens to a hundred of millions of training examples. Our code will be available at https://github.com/google-research/jax-influence.},
  archiveprefix = {arxiv},
  keywords = {notion}
}

@inproceedings{schoch_csshapley_2022,
  title = {{{CS-Shapley}}: {{Class-wise Shapley Values}} for {{Data Valuation}} in {{Classification}}},
  shorttitle = {{{CS-Shapley}}},
  booktitle = {Proc. of the Thirty-Sixth {{Conference}} on {{Neural Information Processing Systems}} ({{NeurIPS}})},
  author = {Schoch, Stephanie and Xu, Haifeng and Ji, Yangfeng},
  year = {2022},
  month = oct,
  address = {{New Orleans, Louisiana, USA}},
  url = {https://openreview.net/forum?id=KTOcrOR5mQ9},
  urldate = {2022-11-23},
  abstract = {Data valuation, or the valuation of individual datum contributions, has seen growing interest in machine learning due to its demonstrable efficacy for tasks such as noisy label detection. In particular, due to the desirable axiomatic properties, several Shapley value approximations have been proposed. In these methods, the value function is usually defined as the predictive accuracy over the entire development set. However, this limits the ability to differentiate between training instances that are helpful or harmful to their own classes. Intuitively, instances that harm their own classes may be noisy or mislabeled and should receive a lower valuation than helpful instances. In this work, we propose CS-Shapley, a Shapley value with a new value function that discriminates between training instances' in-class and out-of-class contributions. Our theoretical analysis shows the proposed value function is (essentially) the unique function that satisfies two desirable properties for evaluating data values in classification. Further, our experiments on two benchmark evaluation tasks (data removal and noisy label detection) and four classifiers demonstrate the effectiveness of CS-Shapley over existing methods. Lastly, we evaluate the ``transferability'' of data values estimated from one classifier to others, and our results suggest Shapley-based data valuation is transferable for application across different models.},
  langid = {english},
  keywords = {notion}
}

@misc{wang_data_2022,
  title = {Data {{Banzhaf}}: {{A Robust Data Valuation Framework}} for {{Machine Learning}}},
  shorttitle = {Data {{Banzhaf}}},
  author = {Wang, Jiachen T. and Jia, Ruoxi},
  year = {2022},
  month = oct,
  number = {arXiv:2205.15466},
  eprint = {arXiv:2205.15466},
  publisher = {{arXiv}},
  doi = {10.48550/arXiv.2205.15466},
  url = {http://arxiv.org/abs/2205.15466},
  urldate = {2022-10-28},
  abstract = {This paper studies the robustness of data valuation to noisy model performance scores. Particularly, we find that the inherent randomness of the widely used stochastic gradient descent can cause existing data value notions (e.g., the Shapley value and the Leave-one-out error) to produce inconsistent data value rankings across different runs. To address this challenge, we first pose a formal framework within which one can measure the robustness of a data value notion. We show that the Banzhaf value, a value notion originated from cooperative game theory literature, achieves the maximal robustness among all semivalues -- a class of value notions that satisfy crucial properties entailed by ML applications. We propose an algorithm to efficiently estimate the Banzhaf value based on the Maximum Sample Reuse (MSR) principle. We derive the lower bound sample complexity for Banzhaf value estimation, and we show that our MSR algorithm's sample complexity is close to the lower bound. Our evaluation demonstrates that the Banzhaf value outperforms the existing semivalue-based data value notions on several downstream ML tasks such as learning with weighted samples and noisy label detection. Overall, our study suggests that when the underlying ML algorithm is stochastic, the Banzhaf value is a promising alternative to the semivalue-based data value schemes given its computational advantage and ability to robustly differentiate data quality.},
  archiveprefix = {arxiv},
  keywords = {notion}
}

@inproceedings{wang_improving_2022,
  title = {Improving {{Cooperative Game Theory-based Data Valuation}} via {{Data Utility Learning}}},
  booktitle = {International {{Conference}} on {{Learning Representations}} ({{ICLR}} 2022). {{Workshop}} on {{Socially Responsible Machine Learning}}},
  author = {Wang, Tianhao and Yang, Yu and Jia, Ruoxi},
  year = {2022},
  month = apr,
  eprint = {2107.06336v2},
  publisher = {{arXiv}},
  doi = {10.48550/arXiv.2107.06336},
  url = {http://arxiv.org/abs/2107.06336v2},
  urldate = {2022-05-19},
  abstract = {The Shapley value (SV) and Least core (LC) are classic methods in cooperative game theory for cost/profit sharing problems. Both methods have recently been proposed as a principled solution for data valuation tasks, i.e., quantifying the contribution of individual datum in machine learning. However, both SV and LC suffer computational challenges due to the need for retraining models on combinatorially many data subsets. In this work, we propose to boost the efficiency in computing Shapley value or Least core by learning to estimate the performance of a learning algorithm on unseen data combinations. Theoretically, we derive bounds relating the error in the predicted learning performance to the approximation error in SV and LC. Empirically, we show that the proposed method can significantly improve the accuracy of SV and LC estimation.},
  archiveprefix = {arxiv},
  langid = {english},
  keywords = {notion}
}

@inproceedings{yan_if_2021,
  title = {If {{You Like Shapley Then You}}'ll {{Love}} the {{Core}}},
  booktitle = {Proceedings of the 35th {{AAAI Conference}} on {{Artificial Intelligence}}, 2021},
  author = {Yan, Tom and Procaccia, Ariel D.},
  year = {2021},
  month = may,
  volume = {6},
  pages = {5751--5759},
  publisher = {{Association for the Advancement of Artificial Intelligence}},
  address = {{Virtual conference}},
  doi = {10.1609/aaai.v35i6.16721},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/16721},
  urldate = {2021-04-23},
  abstract = {The prevalent approach to problems of credit assignment in machine learning \textemdash{} such as feature and data valuation\textemdash{} is to model the problem at hand as a cooperative game and apply the Shapley value. But cooperative game theory offers a rich menu of alternative solution concepts, which famously includes the core and its variants. Our goal is to challenge the machine learning community's current consensus around the Shapley value, and make a case for the core as a viable alternative. To that end, we prove that arbitrarily good approximations to the least core \textemdash{} a core relaxation that is always feasible \textemdash{} can be computed efficiently (but prove an impossibility for a more refined solution concept, the nucleolus). We also perform experiments that corroborate these theoretical results and shed light on settings where the least core may be preferable to the Shapley value.},
  copyright = {Copyright (c) 2021, Association for the Advancement of Artificial Intelligence (www.aaai.org). All rights reserved.},
  langid = {english},
  keywords = {notion}
}
