Function srm_p_value

api/experimentation/stats.py:87–108

Sample ratio mismatch check: was traffic split as configured? Returns the probability that random assignment alone would drift from the configured split at least as much as we observed. A tiny value (< 0.001 by convention) means the split is broken and the results can't be trusted.

(
    observed: Sequence[int],
    expected_shares: Sequence[float],
)

def srm_p_value(
    observed: Sequence[int],
    expected_shares: Sequence[float],
) -> float | None:
    """Sample ratio mismatch check: was traffic split as configured?

    Returns the probability that random assignment alone would drift from the
    configured split at least as much as we observed. A tiny value (< 0.001 by
    convention) means the split is broken and the results can&#x27;t be trusted.
    ``None`` when the question is meaningless (no traffic, one variant).
    """
    total = sum(observed)
    if len(observed) < 2 or total == 0 or any(s <= 0 for s in expected_shares):
        return None

    # Chi-squared statistic: total squared gap between observed and expected
    # counts, scaled by what's expected. Bigger gap == bigger number.
    statistic = sum(
        (count - total * share) ** 2 / (total * share)
        for count, share in zip(observed, expected_shares, strict=True)
    )
    return _chi_squared_survival(statistic, degrees_of_freedom=len(observed) - 1)


def _chi_squared_survival(statistic: float, degrees_of_freedom: int) -> float: