commit a5df9402b611f9fcfcbd22e486ae0ae16246ecd0 Author: George Kadianakis <desnacked@riseup.net> Date: Thu Mar 21 16:49:56 2019 +0200 prob-distr: Decrease false positive rate of stochastic tests. --- changes/bug29693 | 3 +++ src/test/test_prob_distr.c | 15 +++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/changes/bug29693 b/changes/bug29693 new file mode 100644 index 000000000..33ce051c4 --- /dev/null +++ b/changes/bug29693 @@ -0,0 +1,3 @@ + o Minor bugfixes (unit tests): + - Decrease the false positive rate of stochastic probability distribution + tests. Fixes bug 29693; bugfix on 0.4.0.1-alpha. \ No newline at end of file diff --git a/src/test/test_prob_distr.c b/src/test/test_prob_distr.c index 42cc6d70f..37cfdae7d 100644 --- a/src/test/test_prob_distr.c +++ b/src/test/test_prob_distr.c @@ -878,11 +878,14 @@ test_uniform_interval(void *arg) * is higher: 1 - Binom(0; n, alpha) = 1 - (1 - alpha)^n. For n = 10, * this is about 10%, and for n = 100 it is well over 50%. * - * We can drive it down by running each test twice, and accepting it if - * it passes at least once; in that case, it is as if we used Binom(2; - * 2, alpha) = alpha^2 as the false positive rate for each test, and - * for n = 10 tests, it would be 0.1%, and for n = 100 tests, still - * only 1%. + * Given that these tests will run with every CI job, we want to drive down the + * false positive rate. We can drive it down by running each test four times, + * and accepting it if it passes at least once; in that case, it is as if we + * used Binom(4; 2, alpha) = alpha^4 as the false positive rate for each test, + * and for n = 10 tests, it would be 9.99999959506e-08. If each CI build has 14 + * jobs, then the chance of a CI build failing is 1.39999903326e-06, which + * means that a CI build will break with probability 50% after about 495106 + * builds. * * The critical value for a chi^2 distribution with 100 degrees of * freedom and false positive rate alpha = 1% was taken from: @@ -895,7 +898,7 @@ test_uniform_interval(void *arg) static const size_t NSAMPLES = 100000; /* Number of chances we give to the test to succeed. */ -static const unsigned NTRIALS = 2; +static const unsigned NTRIALS = 4; /* Number of times we want the test to pass per NTRIALS. */ static const unsigned NPASSES_MIN = 1;