Coverage for tests/stats/test_diebold_mariano.py: 100%

1"""

2 This module contains unit tests for scores.stats.tests.diebold_mariano_impl

3"""

4 import numpy as np

5 import pytest

6 import xarray as xr

7

8 from scores.stats.statistical_tests.diebold_mariano_impl import (

9 _dm_gamma_hat_k,

10 _dm_test_statistic,

11 _dm_v_hat,

12 _hg_func,

13 _hg_method_stat,

14 _hln_method_stat,

15 diebold_mariano,

16)

17

18

19@pytest.mark.parametrize(

20 (

21 "da_timeseries",

22 "ts_dim",

23 "h_coord",

24 "method",

25 "confidence_level",

26 "statistic_distribution",

27 "error_msg",

28 ),

29 [

30 (

31 xr.DataArray(data=[1, 2], dims=["x"], coords={"x": [0, 1]}),

32 "x",

33 "h",

34 "KEV",

35 -0.4,

36 "t",

37 "`method` must be one of",

38 ),

39 (

40 xr.DataArray(data=[1, 2], dims=["x"], coords={"x": [0, 1]}),

41 "x",

42 "h",

43 "HG",

44 -0.4,

45 "chi_sq",

46 "`statistic_distribution` must be one of",

47 ),

48 (

49 xr.DataArray(data=[1, 2], dims=["x"], coords={"x": [0, 1]}),

50 "x",

51 "h",

52 "HLN",

53 -0.4,

54 "t",

55 "`confidence_level` must be strictly between 0 and 1.",

56 ),

57 (

58 xr.DataArray(data=[1, 2], dims=["x"], coords={"x": [0, 1]}),

59 "x",

60 "h",

61 "HLN",

62 1.0,

63 "t",

64 "`confidence_level` must be strictly between 0 and 1.",

65 ),

66 (

67 xr.DataArray(data=[1, 2], dims=["x"], coords={"x": [0, 1]}),

68 "x",

69 "h",

70 "HLN",

71 0.95,

72 "t",

73 "`da_timeseries` must have exactly two dimensions.",

74 ),

75 (

76 xr.DataArray(data=[[1], [2]], dims=["x", "y"], coords={"x": [0, 1], "y": [1]}),

77 "z",

78 "y",

79 "HLN",

80 0.95,

81 "t",

82 "`ts_dim` 'z' must be a dimension of `da_timeseries`.",

83 ),

84 (

85 xr.DataArray(

86 data=[[1, 2]],

87 dims=["x", "y"],

88 coords={"x": [0], "y": [0, 1], "h": ("x", [1])},

89 ),

90 "x",

91 "h1",

92 "HLN",

93 0.5,

94 "t",

95 "`h_coord` must be among the coordinates of `da_timeseries`.",

96 ),

97 (

98 xr.DataArray(

99 data=[[1, 2]],

100 dims=["x", "y"],

101 coords={"x": [0], "y": [0, 1], "h": ("x", [1.5])},

102 ),

103 "x",

104 "h",

105 "HLN",

106 0.3,

107 "t",

108 " must be an integer.",

109 ),

110 (

111 xr.DataArray(

112 data=[[1, 2]],

113 dims=["x", "y"],

114 coords={"x": [0], "y": [0, 1], "h": ("x", [np.nan])},

115 ),

116 "x",

117 "h",

118 "HLN",

119 0.9,

120 "t",

121 " must be an integer.",

122 ),

123 (

124 xr.DataArray(

125 data=[[1, 2]],

126 dims=["x", "y"],

127 coords={"x": [0], "y": [0, 1], "h": ("x", [-2])},

128 ),

129 "x",

130 "h",

131 "HLN",

132 0.8,

133 "t",

134 " must be positive.",

135 ),

136 (

137 xr.DataArray(

138 data=[[1, 2, 3, 4], [1, 2, np.nan, np.nan]],

139 dims=["x", "y"],

140 coords={"x": [0, 1], "y": [0, 1, 2, 3], "h": ("x", [3, 3])},

141 ),

142 "x",

143 "h",

144 "HLN",

145 0.7,

146 "t",

147 "must be less than the length of the corresponding timeseries",

148 ),

149 (

150 xr.DataArray(

151 data=[[1, 2, 3, 4], [1, 2, np.nan, np.nan]],

152 dims=["x", "y"],

153 coords={"x": [0, 1], "y": [0, 1, 2, 3], "h": ("x", [4, 1])},

154 ),

155 "x",

156 "h",

157 "HLN",

158 0.6,

159 "t",

160 "must be less than the length of the corresponding timeseries",

161 ),

162 ],

163)

164 def test_diebold_mariano_raises(

165 da_timeseries,

166 ts_dim,

167 h_coord,

168 method,

169 confidence_level,

170 statistic_distribution,

171 error_msg,

172):

173 """Tests that diebold_mariano raises a ValueError as expected."""

174 with pytest.raises(ValueError, match=error_msg):

175 diebold_mariano(

176 da_timeseries,

177 ts_dim,

178 h_coord,

179 method,

180 confidence_level,

181 statistic_distribution,

182 )

183

184

185 def test__hg_func():

186 """Tests that _hg_func returns as expected."""

187 pars = [2, 0.5]

188 lag = np.array([0, 1, 2])

189 acv = np.array([1, 2, -1])

190 expected = 4 * np.exp(np.array([0, -6, -12])) - acv

191 result = _hg_func(pars, lag, acv)

192 np.testing.assert_allclose(result, expected)

193

194

195 DM_DIFFS = np.array([1.0, 2, 3, 4])

196 DM_DIFFS2 = np.array([0.0, -1, 0, 2, -1, 0, 1, -4, -1, -1, 2, 1, 0, 2, -1])

197 DM_DIFFS3 = np.array([0.0, -1, 0, 1, -1, 1, -2, -2, -1, -1])

198

199

200@pytest.mark.parametrize(

201 ("diffs", "h", "expected"),

202 [

203 (DM_DIFFS2, 14, -0.169192), # cross-checked with R verification

204 (DM_DIFFS2, 1, -0.169192),

205 (DM_DIFFS3, 10, -1.860521), # cross-checked with R verification

206 ],

207)

208 def test__hg_method_stat1(diffs, h, expected):

209 """

210 Tests that _hg_method_stat returns result as expected when least_squares

211 routine is successful.

212

213 The results of the first test were cross-checked with the R package `verification`.

214 The following R code reproduces the results:

215 library(verification)

216 obs <- rep(0, 15)

217 fcst1 <- c(1, 2, -1, 4, -2, 3, 4, 1, 0, 0, -2, 3, 4, -5, -4)

218 fcst1 <- c(1, 3, 1, 2, -3, 3, 3, 5, 1, -1, 0, 2, 4, -3, -5)

219 test <- predcomp.test(obs, fcst1, fcst2, test = "HG")

220 summary(test)

221 """

222 result = _hg_method_stat(diffs, h)

223 np.testing.assert_allclose(result, expected, atol=1e-5)

224

225

226@pytest.mark.parametrize(

227 ("k", "expected"),

228 [

229 (1, 1.25),

230 (2, -1.5),

231 ],

232)

233 def test__dm_gamma_hat_k(k, expected):

234 """Tests that _dm_gamma_hat_k returns values as expected."""

235 result = _dm_gamma_hat_k(DM_DIFFS, 2.5, 4, k)

236 np.testing.assert_allclose(result, expected)

237

238

239@pytest.mark.parametrize(

240 ("diffs", "h", "expected"),

241 [

242 (DM_DIFFS, 3, (5.0 + 2 * 1.25 + 2 * (-1.5)) / 16),

243 (np.array([1, -1, 1, -1]), 2, np.nan), # original result = 0, so changed to NaN

244 ],

245)

246 def test__dm_v_hat(diffs, h, expected):

247 """Tests that _dm_v_hat returns values as expected."""

248

249 np.testing.assert_allclose(_dm_v_hat(diffs, np.mean(diffs), len(diffs), h), expected)

250

251

252# DM test stat when timeseries is [1, 2, 3, 4] and h = 2

253 DM_TEST_STAT_EXP1 = ((3 / 8) ** 0.5) * 2.5 * (0.46875 ** (-0.5))

254

255

256@pytest.mark.parametrize(

257 ("diffs", "h", "expected"),

258 [

259 (DM_DIFFS, 2, DM_TEST_STAT_EXP1),

260 (np.array([1, -1, 1, -1]), 2, np.nan), # v_hat = 0, so output NaN

261 ],

262)

263 def test__hln_method_stat(diffs, h, expected):

264 """Tests that _hln_method_stat returns as expected."""

265 result = _hln_method_stat(diffs, h)

266 np.testing.assert_allclose(result, expected)

267

268

269@pytest.mark.parametrize(

270 ("diffs", "h", "method", "expected"),

271 [

272 (DM_DIFFS, 2, "HLN", DM_TEST_STAT_EXP1),

273 (np.array([1.0, 1, 1, 1]), 2, "HLN", np.nan),

274 (np.array([0, 0, 0, 0]), 2, "HLN", np.nan),

275 (DM_DIFFS2, 14, "HG", -0.1691921),

276 ],

277)

278 def test__dm_test_statistic(diffs, h, method, expected):

279 """Tests that _dm_test_statistic returns values as expected."""

280 result = _dm_test_statistic(diffs, h, method)

281 np.testing.assert_allclose(result, expected, atol=1e-5)

282

283

284 def test__dm_test_statistic_with_nan():

285 """Tests that _dm_test_statistic returns values as expected with a NaN diffs."""

286 diffs = np.array([np.nan, 1, 2, 3, 4.0, np.nan])

287 with pytest.warns(RuntimeWarning):

288 result = _dm_test_statistic(diffs, 2, "HLN")

289 np.testing.assert_allclose(result, DM_TEST_STAT_EXP1, atol=1e-5)

290

291

292@pytest.mark.filterwarnings("ignore::RuntimeWarning")

293@pytest.mark.parametrize(

294 ("diff", "h", "method", "error_msg"),

295 [

296 (DM_DIFFS, 3, "KEV", "`method` must be one of"),

297 (DM_DIFFS, 100, "HLN", "The condition"),

298 (np.array([np.nan, 1, np.nan]), 2, "HLN", "The condition"),

299 (DM_DIFFS, 0, "HG", "The condition"),

300 ],

301)

302 def test__dm_test_statistic_raises(diff, h, method, error_msg):

303 """Tests that _dm_test_statistic raises a ValueError as expected."""

304 with pytest.raises(ValueError, match=error_msg):

305 _dm_test_statistic(diff, h, method)

306

307

308# DM test stat when timeseries is [2.0, 1, -3, -1, 0] and h = 3

309 DM_TEST_STAT_EXP2 = ((6 / 25) ** 0.5) * (-0.2) * (0.0864 ** (-0.5))

310

311# expected outputs for dm_test_stats

312 DM_TEST_STATS_T_EXP = xr.Dataset(

313 data_vars=dict(

314 mean=(["lead_day"], [2.5, -0.2, 1.0]),

315 dm_test_stat=(["lead_day"], [DM_TEST_STAT_EXP1, DM_TEST_STAT_EXP2, np.nan]),

316 timeseries_len=(["lead_day"], [4, 5, 5]),

317 confidence_gt_0=(

318 ["lead_day"],

319 [0.9443164226429581, 0.3778115634892615, np.nan],

320 ),

321 ci_upper=(["lead_day"], [5.131140307989639, 1.079108068801774, np.nan]),

322 ci_lower=(

323 ["lead_day"],

324 [-0.13114030798963894, -1.4791080688017741, np.nan],

325 ),

326 ),

327 coords={"lead_day": [1, 2, 3]},

328)

329

330 DM_TEST_STATS_NORMAL_EXP = xr.Dataset(

331 data_vars=dict(

332 mean=(["lead_day"], [2.5, -0.2, 1.0]),

333 dm_test_stat=(["lead_day"], [DM_TEST_STAT_EXP1, DM_TEST_STAT_EXP2, np.nan]),

334 timeseries_len=(["lead_day"], [4, 5, 5]),

335 confidence_gt_0=(

336 ["lead_day"],

337 [0.9873263406612659, 0.36944134018176367, np.nan],

338 ),

339 ci_upper=(["lead_day"], [4.339002261450286, 0.7869121761708835, np.nan]),

340 ci_lower=(

341 ["lead_day"],

342 [0.6609977385497137, -1.1869121761708834, np.nan],

343 ),

344 ),

345 coords={"lead_day": [1, 2, 3]},

346)

347

348

349@pytest.mark.parametrize(

350 ("distribution", "expected"),

351 [

352 ("t", DM_TEST_STATS_T_EXP),

353 ("normal", DM_TEST_STATS_NORMAL_EXP),

354 ],

355)

356 def test_diebold_mariano(distribution, expected):

357 """

358 Tests that diebold_mariano gives results as expected and raises a warning

359 due to a NaN in the data.

360 """

361 da_timeseries = xr.DataArray(

362 data=[[1, 2, 3.0, 4, np.nan], [2.0, 1, -3, -1, 0], [1.0, 1, 1, 1, 1]],

363 dims=["lead_day", "valid_date"],

364 coords={

365 "lead_day": [1, 2, 3],

366 "valid_date": ["a", "b", "c", "d", "e"],

367 "h": ("lead_day", [2, 3, 4]),

368 },

369 )

370 with pytest.warns(RuntimeWarning):

371 result = diebold_mariano(

372 da_timeseries,

373 "lead_day",

374 "h",

375 method="HLN",

376 confidence_level=0.9,

377 statistic_distribution=distribution,

378 )

379 xr.testing.assert_allclose(result, expected, atol=7)