<html xmlns:py="http://genshi.edgewall.org/">
		<head>
				<link rel="stylesheet" href="static/swab.css"/>
		</head>
		<body class="pageFullWidth">
		<h1>A/B test results summary</h1>
		<section py:for="exp in experiments">
				<h2>Experiment: $exp.name</h2>
				<?python
				expdata = data[exp.name]
				goals = data[exp.name]['goals']
				control = data[exp.name]['control']
				control_data = data[exp.name]['variants'][control]
				?>
				<table class="data">
						<tr>
								<th rowspan="2">Variant</th>
								<th rowspan="2">Number of tests</th>
								<th py:for="goal in goals" colspan="4">Goal: $goal</th>
						</tr>
						<tr>
								<py:for each="goal in goals">
										<th>conversions</th>
										<th>rate</th>
										<th>z-score</th>
										<th>confidence</th>
								</py:for>
						</tr>
						<tr
								py:for="variant, vdata in sorted(data[exp.name]['variants'].items())"
								class="${'control' if variant==control else None}"
						>
								<th>
										<a href="${request.make_uri(query={'control.{0}'.format(exp.name): variant})}">$variant</a>
										<span class='control' title="This variant is selected as the control: the performance of other variants will be measured against this one" py:if="variant==control">control</span>
								</th>
								<td>${vdata['t']}</td>
								<py:for each="goal in goals">
										<py:if test="not vdata['t']">
												<td colspan="4">no data yet</td>
										</py:if>
										<py:if test="vdata['t']">
												<td>${vdata['c'][goal]}</td>
												<td>${'{0:.1%}'.format(vdata['r'][goal])}</td>
												<td><py:if test="variant != control">${'{0:.3g}'.format(vdata['z'][goal])}</py:if></td>
												<td><py:if test="variant != control">${'{0:.1%}'.format(vdata['confidence'][goal])}</py:if></td>
										</py:if>
								</py:for>
						</tr>
				</table>
				<div py:for="variant, vdata in sorted(data[exp.name]['variants'].items())">
					<py:if test="variant != data[exp.name]['control']">
							<py:for each="goal in goals">
									<py:choose>
											<py:when test="control_data['r'][goal] == 0">
													Insufficient data
											</py:when>
											<py:otherwise>
													<?python
													difference = vdata['r'][goal] / control_data['r'][goal]
													?>
													<p>
															For the <strong>$goal</strong> goal, the <strong>$variant</strong> variant resulted in
															<py:choose>
																	<span py:when="difference &gt;= 1" class="better">${'{0:.2%}'.format(difference - 1)} more conversions than the $control variant</span>
																	<span py:otherwise="" class="worse">${'{0:.2%}'.format(difference - 1)} fewer conversions than the $control variant</span>
															</py:choose>
													</p>
													<ul class="simpleList" py:choose="">
																<div class="confident99" py:when="vdata['confidence'][goal] &gt;= 0.99">
																		<li>
																				You have now collected enough data to be 99% certain that this conclusion is valid.
																		</li>
																		<li class="recommendation">
																				<py:choose>
																						Recommendation:
																						<span py:when="difference &gt;= 1">adopt $variant variant.</span>
																						<span py:otherwise="">adopt $control variant.</span>
																				</py:choose>
																		</li>
																</div>
																<div class="confident95" py:when="vdata['confidence'][goal] &gt;= 0.95">
																		<li>
																				You have collected enough data to be 95% sure that this difference is not simply due to chance factors.
																		</li>
																		<li class="recommendation">
																				Recommendation: <strong>let the test run for longer to be completely certain</strong>.
																		</li>
																</div>
																<div class="unconfident" py:otherwise="">
																		<li>
																				You don't have enough data to be sure yet that this difference is due to the variant you are testing or purely chance factors.
																		</li>
																		<li class="recommendation">
																				Recommendation: <strong>let the test run to collect more data</strong>.
																		</li>
																</div>
													</ul>
											</py:otherwise>
									</py:choose>
							</py:for>
					</py:if>
				</div>


		</section>
		</body>
</html>

