Coverage for /Users/sebastiana/Documents/Sugarpills/confidence/spotify_confidence/analysis/abstract_base_classes/confidence_abc.py: 84%

1# Copyright 2017-2020 Spotify AB

2#

3# Licensed under the Apache License, Version 2.0 (the "License");

4# you may not use this file except in compliance with the License.

5# You may obtain a copy of the License at

6#

7# http://www.apache.org/licenses/LICENSE-2.0

8#

9# Unless required by applicable law or agreed to in writing, software

10# distributed under the License is distributed on an "AS IS" BASIS,

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

12# See the License for the specific language governing permissions and

13# limitations under the License.

14

15 from abc import ABC, abstractmethod

16 from typing import Union, Iterable, Tuple, List

17

18 from pandas import DataFrame

19

20 from spotify_confidence.chartgrid import ChartGrid

21 from .confidence_computer_abc import ConfidenceComputerABC

22 from .confidence_grapher_abc import ConfidenceGrapherABC

23 from ..constants import NIM_TYPE

24

25

26 class ConfidenceABC(ABC):

27 @property

28 def _confidence_computer(self) -> ConfidenceComputerABC:

29 return self._computer

30

31 @_confidence_computer.setter

32 def _confidence_computer(self, computer: ConfidenceComputerABC):

33 self._computer = computer

34

35 @property

36 def _confidence_grapher(self) -> ConfidenceGrapherABC:

37 return self._grapher

38

39 @_confidence_grapher.setter

40 def _confidence_grapher(self, grapher: ConfidenceGrapherABC):

41 self._grapher = grapher

42

43 @abstractmethod

44 def __init__(

45 self,

46 data_frame: DataFrame,

47 numerator_column: str,

48 numerator_sum_squares_column: Union[str, None],

49 denominator_column: str,

50 categorical_group_columns: Union[str, Iterable, None],

51 ordinal_group_column: Union[str, None],

52 interval_size: float,

53 correction_method: str,

54 metric_column: Union[str, None],

55 treatment_column: Union[str, None],

56 power: float,

57 ):

58 pass

59

60 @abstractmethod

61 def summary(self, verbose: bool) -> DataFrame:

62 """Args:

63 verbose (bool): include columns used in intermediate steps in the calculations in returned dataframe.

64 Returns:

65 Dataframe containing summary statistics

66 """

67 pass

68

69 @abstractmethod

70 def difference(

71 self,

72 level_1: Union[str, Tuple],

73 level_2: Union[str, Tuple],

74 absolute: bool,

75 groupby: Union[str, Iterable],

76 non_inferiority_margins: NIM_TYPE,

77 final_expected_sample_size_column: str,

78 verbose: bool,

79 minimum_detectable_effects_column: str,

80 ) -> DataFrame:

81 """Args:

82 groupby (str): Name of column.

83 If specified, will plot a separate chart for each level of the

84 grouping.

85 non_inferiority_margins (Union[Tuple[float, str], Dict[str, Tuple[float, str]], bool]):

86 Pass tuple(non_inferiority_margin, preferred direction) to use the same NIM for all

87 comparisons, e.g. (0.01, 'increase'), which means that we want

88 level_2 to be grater than the average of level_1 times (1-0.01),

89 or (0.05, 'decrease') which means that we want

90 level_2 to be smaller than the average

91 of level_1 times (1+0.01).

92 Pass dictionary {{group:tuple(non_inferiority_margin, preferred direction}} to use

93 different non-inferiority margins for different values of

94 groupby column.

95 To performe a one-sided test without nim, use

96 (None, preffered direction).

97 Alternatively, pass True to use the "non_inferiority_margin" and "preferred_direction"

98 columns of dataframe that was passed to the contructor, as source of nims.

99 final_expected_sample_size_column (str): Column in source data frame containing expected number of

100 observations at end of experiment.

101 Use in combination with ordinal groupby to perform a

102 sequential test. See https://cran.r-project.org/web/packages/ldbounds/index.html for details.

103 verbose (bool): include columns used in intermediate steps in the calculations in returned dataframe.

104 minimum_detectable_effects_column (str): The minimum detectable effect, used for calculating required

105 sample size.

106

107 Returns:

108 Dataframe containing the difference in means between

109 group 1 and 2, p-values and confidence intervals for each value

110 in the groupby column

111 """

112 pass

113

114 @abstractmethod

115 def differences(

116 self,

117 levels: List[Tuple],

118 absolute: bool,

119 groupby: Union[str, Iterable],

120 non_inferiority_margins: NIM_TYPE,

121 final_expected_sample_size_column: str,

122 verbose: bool,

123 minimum_detectable_effects_column: str,

124 ) -> DataFrame:

125 """Args:

126 levels: (list(tuple)): list of levels to compare

127 groupby (str): Name of column.

128 If specified, will plot a separate chart for each level of the

129 grouping.

130 non_inferiority_margins (Union[Tuple[float, str], Dict[str, Tuple[float, str]], bool]):

131 Pass tuple(non_inferiority_margin, preferred direction) to use the same NIM for all

132 comparisons, e.g. (0.01, 'increase'), which means that we want

133 level_2 to be grater than the average of level_1 times (1-0.01),

134 or (0.05, 'decrease') which means that we want

135 level_2 to be smaller than the average

136 of level_1 times (1+0.01).

137 Pass dictionary {{group:tuple(non_inferiority_margin, preferred direction}} to use

138 different non-inferiority margins for different values of

139 groupby column.

140 To performe a one-sided test without nim, use

141 (None, preffered direction).

142 Alternatively, pass True to use the "non_inferiority_margin" and "preferred_direction"

143 columns of dataframe that was passed to the contructor, as source of nims.

144 final_expected_sample_size_column (str): Column in source data frame containing expected number of

145 observations at end of experiment.

146 Use in combination with ordinal groupby to perform a

147 sequential test. See https://cran.r-project.org/web/packages/ldbounds/index.html for details.

148 verbose (bool): include columns used in intermediate steps in the calculations in returned dataframe.

149 minimum_detectable_effects_column (str): The minimum detectable effect, used for calculating required

150 sample size.

151

152 Returns:

153 Dataframe containing the difference in means between

154 group 1 and 2, p-values and confidence intervals for each value

155 in the groupby column

156 """

157 pass

158

159 @abstractmethod

160 def multiple_difference(

161 self,

162 level: Union[str, Tuple],

163 absolute: bool,

164 groupby: Union[str, Iterable],

165 level_as_reference: bool,

166 non_inferiority_margins: NIM_TYPE,

167 final_expected_sample_size_column: str,

168 verbose: bool,

169 minimum_detectable_effects_column: str,

170 ) -> DataFrame:

171 """Args:

172 groupby (str): Name of column.

173 If specified, will plot a separate chart for each level of the

174 grouping.

175 level_as_reference (bool):

176 If false (default), compare level to all other

177 groups. If true, compare all other groups to level.

178 non_inferiority_margins (Union[Tuple[float, str],

179 Dict[str, Tuple[float, str]]]):

180 Pass tuple(nim, preferred direction) to use the same NIM for all

181 comparisons, e.g. (0.01, 'increase'), which means that we want

182 level_2 to be grater than the average of level_1 times (1-0.01),

183 or (0.05, 'decrease') which means that we want

184 level_2 to be smaller than the average

185 of level_1 times (1+0.01).

186 Pass dictionary {{group:tuple(nim, preferred direction}} to use

187 different non-inferiority margins for different values of

188 groupby column.

189 To performe a one-sided test without nim, use

190 (None, preffered direction).

191 Alternatively, pass True to use the "non_inferiority_margin" and "preferred_direction"

192 columns of dataframe that was passed to the contructor, as source of nims.

193 final_expected_sample_size_column (str): Column in source data frame containing expected number of

194 observations at end of experiment.

195 Use in combination with ordinal groupby to perform a

196 sequential test. See https://cran.r-project.org/web/packages/ldbounds/index.html for details.

197 verbose (bool): include columns used in intermediate steps in the calculations in returned dataframe.

198 minimum_detectable_effects_column (str): The minimum detectable effect, used for calculating required

199 sample size.

200

201 Returns:

202 Dataframe containing the difference in means between

203 group 1 and 2, p-values and confidence intervals for each value

204 in the groupby column

205 """

206 pass

207

208 @abstractmethod

209 def summary_plot(self, groupby: Union[str, Iterable]) -> ChartGrid:

210 """Plot for each group in the data_frame:

211

212 if ordinal level exists:

213 line graph with area to represent confidence interval

214 if categorical levels:

215 Interval plots of confidence intervals by group

216

217 Args:

218 groupby (str): Name of column.

219 If specified, will plot a separate chart for each level of the

220 grouping.

221

222 Returns:

223 ChartGrid object and a DataFrame with numerical results.

224 """

225 pass

226

227 @abstractmethod

228 def difference_plot(

229 self,

230 level_1: Union[str, Tuple],

231 level_2: Union[str, Tuple],

232 absolute: bool,

233 groupby: Union[str, Iterable],

234 non_inferiority_margins: NIM_TYPE,

235 use_adjusted_intervals: bool,

236 final_expected_sample_size_column: str,

237 ) -> ChartGrid:

238 """Plot representing the difference between group 1 and 2.

239 - Difference in means or proportions, depending

240 on the response variable type.

241

242 - Plot interval plot with confidence interval of the

243 difference between groups

244

245 Args:

246 level_1 (str, tuple of str): Name of first level.

247 level_2 (str, tuple of str): Name of second level.

248 absolute (bool): If True then return the absolute

249 difference (level2 - level1)

250 otherwise return the relative difference (level2 / level1 - 1)

251 groupby (str): Name of column, or list of columns.

252 If specified, will return an interval for each level

253 of the grouped dimension, or a confidence band if the

254 grouped dimension is ordinal

255 non_inferiority_margins (Union[Tuple[float, str],

256 Dict[str, Tuple[float, str]]]):

257 Pass tuple(nim, preferred direction) to use the same NIM for all

258 comparisons, e.g. (0.01, 'increase'), which means that we want

259 level_2 to be grater than the average of level_1 times (1-0.01),

260 or (0.05, 'decrease') which means that we want

261 level_2 to be smaller than the average

262 of level_1 times (1+0.01).

263 Pass dictionary {{group:tuple(nim, preferred direction}} to use

264 different non-inferiority margins for different values of

265 groupby column.

266 To performe a one-sided test without nim, use

267 (None, preffered direction).

268 use_adjusted_intervals (bool):

269 If true, use e.g. bon-ferroni corrected

270 (or other method provided) confidence intervals

271 final_expected_sample_size_column (str): Column in source data frame containing expected number of

272 observations at end of experiment.

273 Use in combination with ordinal groupby to perform a

274 sequential test. See https://cran.r-project.org/web/packages/ldbounds/index.html for details.

275

276 Returns:

277 Chartify Chart object and a DataFrame with numerical results.

278 """

279

280 @abstractmethod

281 def differences_plot(

282 self,

283 levels: List[Tuple],

284 absolute: bool,

285 groupby: Union[str, Iterable],

286 non_inferiority_margins: NIM_TYPE,

287 use_adjusted_intervals: bool,

288 final_expected_sample_size_column: str,

289 ) -> ChartGrid:

290 """Plot representing the difference between group 1 and 2.

291 - Difference in means or proportions, depending

292 on the response variable type.

293

294 - Plot interval plot with confidence interval of the

295 difference between groups

296

297 Args:

298 levels: (list(tuple)): list of levels to compare

299 absolute (bool): If True then return the absolute

300 difference (level2 - level1)

301 otherwise return the relative difference (level2 / level1 - 1)

302 groupby (str): Name of column, or list of columns.

303 If specified, will return an interval for each level

304 of the grouped dimension, or a confidence band if the

305 grouped dimension is ordinal

306 non_inferiority_margins (Union[Tuple[float, str],

307 Dict[str, Tuple[float, str]]]):

308 Pass tuple(nim, preferred direction) to use the same NIM for all

309 comparisons, e.g. (0.01, 'increase'), which means that we want

310 level_2 to be grater than the average of level_1 times (1-0.01),

311 or (0.05, 'decrease') which means that we want

312 level_2 to be smaller than the average

313 of level_1 times (1+0.01).

314 Pass dictionary {{group:tuple(nim, preferred direction}} to use

315 different non-inferiority margins for different values of

316 groupby column.

317 To performe a one-sided test without nim, use

318 (None, preffered direction).

319 use_adjusted_intervals (bool):

320 If true, use e.g. bon-ferroni corrected

321 (or other method provided) confidence intervals

322 final_expected_sample_size_column (str): Column in source data frame containing expected number of

323 observations at end of experiment.

324 Use in combination with ordinal groupby to perform a

325 sequential test. See https://cran.r-project.org/web/packages/ldbounds/index.html for details.

326

327 Returns:

328 Chartify Chart object and a DataFrame with numerical results.

329 """

330

331 @abstractmethod

332 def multiple_difference_plot(

333 self,

334 level: Union[str, Tuple],

335 absolute: bool,

336 groupby: Union[str, Iterable],

337 level_as_reference: bool,

338 non_inferiority_margins: NIM_TYPE,

339 use_adjusted_intervals: bool,

340 final_expected_sample_size_column: str,

341 ) -> ChartGrid:

342 """Compare level to all other groups or, if level_as_reference = True,

343 all other groups to level.

344

345 Args:

346 level (str, tuple of str): Name of level.

347 absolute (bool): If True then return the absolute

348 difference (level2 - level1)

349 otherwise return the relative difference (level2 / level1 - 1)

350 groupby (str): Name of column, or list of columns.

351 If specified, will return an interval for each level

352 of the grouped dimension, or a confidence band if the

353 grouped dimension is ordinal

354 level_as_reference: If false (default), compare level to all other

355 groups. If true, compare all other groups to level.

356 non_inferiority_margins (Union[Tuple[float, str],

357 Dict[str, Tuple[float, str]]]):

358 Pass tuple(nim, preferred direction) to use the same NIM for all

359 comparisons, e.g. (0.01, 'increase'), which means that we want

360 level_2 to be grater than the average of level_1 times (1-0.01),

361 or (0.05, 'decrease') which means that we want

362 level_2 to be smaller than the average

363 of level_1 times (1+0.01).

364 Pass dictionary {{group:tuple(nim, preferred direction}} to use

365 different non-inferiority margins for different values of

366 groupby column.

367 To performe a one-sided test without nim, use

368 (None, preffered direction).

369 use_adjusted_intervals (bool):

370 If true, use e.g. bon-ferroni corrected

371 (or other method provided) confidence intervals

372 final_expected_sample_size_column (str): Column in source data frame containing expected number of

373 observations at end of experiment.

374 Use in combination with ordinal groupby to perform a

375 sequential test. See https://cran.r-project.org/web/packages/ldbounds/index.html for details.

376

377 Returns:

378 ChartGrid object and a DataFrame with numerical results.

379 """