test/bntest.c:2096: error: BUFFER_OVERRUN_L3
  Offset: 0 Size: [0, 8388607] by call to `BN_is_prime_fasttest_ex`.
Showing all 10 steps of the trace


test/bntest.c:2095:10: Call
2093.        * do_trial_division was set. */
2094.       if (r == NULL ||
2095.           !BN_set_word(r, 3) ||
                 ^
2096.           BN_is_prime_fasttest_ex(r, 3 /* nchecks */, ctx,
2097.                                   0 /* do_trial_division */, NULL) != 1 ||

crypto/bn/bn_lib.c:395:1: Parameter `*a->d`
  393.   }
  394.   
  395. > int BN_set_word(BIGNUM *a, BN_ULONG w)
  396.   {
  397.       bn_check_top(a);

crypto/bn/bn_lib.c:398:9: Call
  396.   {
  397.       bn_check_top(a);
  398.       if (bn_expand(a, (int)sizeof(BN_ULONG) * 8) == NULL)
                 ^
  399.           return (0);
  400.       a->neg = 0;

crypto/bn/bn_lcl.h:660:1: Parameter `*a->d`
    658.                            const BIGNUM *add, const BIGNUM *rem, BN_CTX *ctx);
    659.   
    660. > static ossl_inline BIGNUM *bn_expand(BIGNUM *a, int bits)
    661.   {
    662.       if (bits > (INT_MAX - BN_BITS2 + 1))

test/bntest.c:2096:9: Call
2094.       if (r == NULL ||
2095.           !BN_set_word(r, 3) ||
2096.           BN_is_prime_fasttest_ex(r, 3 /* nchecks */, ctx,
                ^
2097.                                   0 /* do_trial_division */, NULL) != 1 ||
2098.           BN_is_prime_fasttest_ex(r, 3 /* nchecks */, ctx,

crypto/bn/bn_prime.c:153:1: Parameter `*a->d`
  151.   }
  152.   
  153. > int BN_is_prime_fasttest_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed,
  154.                               int do_trial_division, BN_GENCB *cb)
  155.   {

crypto/bn/bn_prime.c:229:10: Call
  227.       if (mont == NULL)
  228.           goto err;
  229.       if (!BN_MONT_CTX_set(mont, A, ctx))
                  ^
  230.           goto err;
  231.   

crypto/bn/bn_mont.c:247:1: <Length trace>
245.   }
246.   
247. > int BN_MONT_CTX_set(BN_MONT_CTX *mont, const BIGNUM *mod, BN_CTX *ctx)
248.   {
249.       int ret = 0;

crypto/bn/bn_mont.c:247:1: Parameter `*mod->d`
    245.   }
    246.   
    247. > int BN_MONT_CTX_set(BN_MONT_CTX *mont, const BIGNUM *mod, BN_CTX *ctx)
    248.   {
    249.       int ret = 0;

crypto/bn/bn_mont.c:322:18: Array access: Offset: 0 Size: [0, 8388607] by call to `BN_is_prime_fasttest_ex` 
    320.               goto err;           /* R */
    321.   
    322.           buf[0] = mod->d[0];     /* tmod = N mod word size */
                            ^
    323.           buf[1] = 0;
    324.           tmod.top = buf[0] != 0 ? 1 : 0;

int BN_MONT_CTX_set(BN_MONT_CTX *mont, const BIGNUM *mod, BN_CTX *ctx)
{
    int ret = 0;
    BIGNUM *Ri, *R;
    if (BN_is_zero(mod))
        return 0;
    BN_CTX_start(ctx);
    if ((Ri = BN_CTX_get(ctx)) == NULL)
        goto err;
    R = &(mont->RR);
    if (!BN_copy(&(mont->N), mod))
        goto err;
    mont->N.neg = 0;
#ifdef MONT_WORD
    {
        BIGNUM tmod;
        BN_ULONG buf[2];
        bn_init(&tmod);
        tmod.d = buf;
        tmod.dmax = 2;
        tmod.neg = 0;
        mont->ri = (BN_num_bits(mod) + (BN_BITS2 - 1)) / BN_BITS2 * BN_BITS2;
# if defined(OPENSSL_BN_ASM_MONT) && (BN_BITS2<=32)
        BN_zero(R);
        if (!(BN_set_bit(R, 2 * BN_BITS2)))
            goto err;
        tmod.top = 0;
        if ((buf[0] = mod->d[0]))
            tmod.top = 1;
        if ((buf[1] = mod->top > 1 ? mod->d[1] : 0))
            tmod.top = 2;
        if ((BN_mod_inverse(Ri, R, &tmod, ctx)) == NULL)
            goto err;
        if (!BN_lshift(Ri, Ri, 2 * BN_BITS2))
            goto err;
        if (!BN_is_zero(Ri)) {
            if (!BN_sub_word(Ri, 1))
                goto err;
        } else {
            if (bn_expand(Ri, (int)sizeof(BN_ULONG) * 2) == NULL)
                goto err;
            Ri->neg = 0;
            Ri->d[0] = BN_MASK2;
            Ri->d[1] = BN_MASK2;
            Ri->top = 2;
        }
        if (!BN_div(Ri, NULL, Ri, &tmod, ctx))
            goto err;
        mont->n0[0] = (Ri->top > 0) ? Ri->d[0] : 0;
        mont->n0[1] = (Ri->top > 1) ? Ri->d[1] : 0;
# else
        BN_zero(R);
        if (!(BN_set_bit(R, BN_BITS2)))
            goto err;
        buf[0] = mod->d[0];
        buf[1] = 0;
        tmod.top = buf[0] != 0 ? 1 : 0;
        if ((BN_mod_inverse(Ri, R, &tmod, ctx)) == NULL)
            goto err;
        if (!BN_lshift(Ri, Ri, BN_BITS2))
            goto err;
        if (!BN_is_zero(Ri)) {
            if (!BN_sub_word(Ri, 1))
                goto err;
        } else {
            if (!BN_set_word(Ri, BN_MASK2))
                goto err;
        }
        if (!BN_div(Ri, NULL, Ri, &tmod, ctx))
            goto err;
        mont->n0[0] = (Ri->top > 0) ? Ri->d[0] : 0;
        mont->n0[1] = 0;
# endif
    }
#else
    {
        mont->ri = BN_num_bits(&mont->N);
        BN_zero(R);
        if (!BN_set_bit(R, mont->ri))
            goto err;
        if ((BN_mod_inverse(Ri, R, &mont->N, ctx)) == NULL)
            goto err;
        if (!BN_lshift(Ri, Ri, mont->ri))
            goto err;
        if (!BN_sub_word(Ri, 1))
            goto err;
        if (!BN_div(&(mont->Ni), NULL, Ri, &mont->N, ctx))
            goto err;
    }
#endif
    BN_zero(&(mont->RR));
    if (!BN_set_bit(&(mont->RR), mont->ri * 2))
        goto err;
    if (!BN_mod(&(mont->RR), &(mont->RR), &(mont->N), ctx))
        goto err;
    ret = 1;
 err:
    BN_CTX_end(ctx);
    return ret;
}
['static int test_3_is_prime()\n{\n    int ret = 0;\n    BIGNUM *r = BN_new();\n    if (r == NULL ||\n        !BN_set_word(r, 3) ||\n        BN_is_prime_fasttest_ex(r, 3  , ctx,\n                                0  , NULL) != 1 ||\n        BN_is_prime_fasttest_ex(r, 3  , ctx,\n                                1  , NULL) != 1) {\n        goto err;\n    }\n    ret = 1;\nerr:\n    BN_free(r);\n    return ret;\n}', 'int BN_set_word(BIGNUM *a, BN_ULONG w)\n{\n    bn_check_top(a);\n    if (bn_expand(a, (int)sizeof(BN_ULONG) * 8) == NULL)\n        return (0);\n    a->neg = 0;\n    a->d[0] = w;\n    a->top = (w ? 1 : 0);\n    bn_check_top(a);\n    return (1);\n}', 'static ossl_inline BIGNUM *bn_expand(BIGNUM *a, int bits)\n{\n    if (bits > (INT_MAX - BN_BITS2 + 1))\n        return NULL;\n    if (((bits+BN_BITS2-1)/BN_BITS2) <= (a)->dmax)\n        return a;\n    return bn_expand2((a),(bits+BN_BITS2-1)/BN_BITS2);\n}', 'int BN_is_prime_fasttest_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed,\n                            int do_trial_division, BN_GENCB *cb)\n{\n    int i, j, ret = -1;\n    int k;\n    BN_CTX *ctx = NULL;\n    BIGNUM *A1, *A1_odd, *check;\n    BN_MONT_CTX *mont = NULL;\n    const BIGNUM *A = NULL;\n    if (BN_cmp(a, BN_value_one()) <= 0)\n        return 0;\n    if (checks == BN_prime_checks)\n        checks = BN_prime_checks_for_size(BN_num_bits(a));\n    if (!BN_is_odd(a))\n        return BN_is_word(a, 2);\n    if (do_trial_division) {\n        for (i = 1; i < NUMPRIMES; i++) {\n            BN_ULONG mod = BN_mod_word(a, primes[i]);\n            if (mod == (BN_ULONG)-1)\n                goto err;\n            if (mod == 0)\n                return BN_is_word(a, primes[i]);\n        }\n        if (!BN_GENCB_call(cb, 1, -1))\n            goto err;\n    }\n    if (ctx_passed != NULL)\n        ctx = ctx_passed;\n    else if ((ctx = BN_CTX_new()) == NULL)\n        goto err;\n    BN_CTX_start(ctx);\n    if (a->neg) {\n        BIGNUM *t;\n        if ((t = BN_CTX_get(ctx)) == NULL)\n            goto err;\n        if (BN_copy(t, a) == NULL)\n            goto err;\n        t->neg = 0;\n        A = t;\n    } else\n        A = a;\n    A1 = BN_CTX_get(ctx);\n    A1_odd = BN_CTX_get(ctx);\n    check = BN_CTX_get(ctx);\n    if (check == NULL)\n        goto err;\n    if (!BN_copy(A1, A))\n        goto err;\n    if (!BN_sub_word(A1, 1))\n        goto err;\n    if (BN_is_zero(A1)) {\n        ret = 0;\n        goto err;\n    }\n    k = 1;\n    while (!BN_is_bit_set(A1, k))\n        k++;\n    if (!BN_rshift(A1_odd, A1, k))\n        goto err;\n    mont = BN_MONT_CTX_new();\n    if (mont == NULL)\n        goto err;\n    if (!BN_MONT_CTX_set(mont, A, ctx))\n        goto err;\n    for (i = 0; i < checks; i++) {\n        if (!BN_pseudo_rand_range(check, A1))\n            goto err;\n        if (!BN_add_word(check, 1))\n            goto err;\n        j = witness(check, A, A1, A1_odd, k, ctx, mont);\n        if (j == -1)\n            goto err;\n        if (j) {\n            ret = 0;\n            goto err;\n        }\n        if (!BN_GENCB_call(cb, 1, i))\n            goto err;\n    }\n    ret = 1;\n err:\n    if (ctx != NULL) {\n        BN_CTX_end(ctx);\n        if (ctx_passed == NULL)\n            BN_CTX_free(ctx);\n    }\n    BN_MONT_CTX_free(mont);\n    return (ret);\n}', 'int BN_MONT_CTX_set(BN_MONT_CTX *mont, const BIGNUM *mod, BN_CTX *ctx)\n{\n    int ret = 0;\n    BIGNUM *Ri, *R;\n    if (BN_is_zero(mod))\n        return 0;\n    BN_CTX_start(ctx);\n    if ((Ri = BN_CTX_get(ctx)) == NULL)\n        goto err;\n    R = &(mont->RR);\n    if (!BN_copy(&(mont->N), mod))\n        goto err;\n    mont->N.neg = 0;\n#ifdef MONT_WORD\n    {\n        BIGNUM tmod;\n        BN_ULONG buf[2];\n        bn_init(&tmod);\n        tmod.d = buf;\n        tmod.dmax = 2;\n        tmod.neg = 0;\n        mont->ri = (BN_num_bits(mod) + (BN_BITS2 - 1)) / BN_BITS2 * BN_BITS2;\n# if defined(OPENSSL_BN_ASM_MONT) && (BN_BITS2<=32)\n        BN_zero(R);\n        if (!(BN_set_bit(R, 2 * BN_BITS2)))\n            goto err;\n        tmod.top = 0;\n        if ((buf[0] = mod->d[0]))\n            tmod.top = 1;\n        if ((buf[1] = mod->top > 1 ? mod->d[1] : 0))\n            tmod.top = 2;\n        if ((BN_mod_inverse(Ri, R, &tmod, ctx)) == NULL)\n            goto err;\n        if (!BN_lshift(Ri, Ri, 2 * BN_BITS2))\n            goto err;\n        if (!BN_is_zero(Ri)) {\n            if (!BN_sub_word(Ri, 1))\n                goto err;\n        } else {\n            if (bn_expand(Ri, (int)sizeof(BN_ULONG) * 2) == NULL)\n                goto err;\n            Ri->neg = 0;\n            Ri->d[0] = BN_MASK2;\n            Ri->d[1] = BN_MASK2;\n            Ri->top = 2;\n        }\n        if (!BN_div(Ri, NULL, Ri, &tmod, ctx))\n            goto err;\n        mont->n0[0] = (Ri->top > 0) ? Ri->d[0] : 0;\n        mont->n0[1] = (Ri->top > 1) ? Ri->d[1] : 0;\n# else\n        BN_zero(R);\n        if (!(BN_set_bit(R, BN_BITS2)))\n            goto err;\n        buf[0] = mod->d[0];\n        buf[1] = 0;\n        tmod.top = buf[0] != 0 ? 1 : 0;\n        if ((BN_mod_inverse(Ri, R, &tmod, ctx)) == NULL)\n            goto err;\n        if (!BN_lshift(Ri, Ri, BN_BITS2))\n            goto err;\n        if (!BN_is_zero(Ri)) {\n            if (!BN_sub_word(Ri, 1))\n                goto err;\n        } else {\n            if (!BN_set_word(Ri, BN_MASK2))\n                goto err;\n        }\n        if (!BN_div(Ri, NULL, Ri, &tmod, ctx))\n            goto err;\n        mont->n0[0] = (Ri->top > 0) ? Ri->d[0] : 0;\n        mont->n0[1] = 0;\n# endif\n    }\n#else\n    {\n        mont->ri = BN_num_bits(&mont->N);\n        BN_zero(R);\n        if (!BN_set_bit(R, mont->ri))\n            goto err;\n        if ((BN_mod_inverse(Ri, R, &mont->N, ctx)) == NULL)\n            goto err;\n        if (!BN_lshift(Ri, Ri, mont->ri))\n            goto err;\n        if (!BN_sub_word(Ri, 1))\n            goto err;\n        if (!BN_div(&(mont->Ni), NULL, Ri, &mont->N, ctx))\n            goto err;\n    }\n#endif\n    BN_zero(&(mont->RR));\n    if (!BN_set_bit(&(mont->RR), mont->ri * 2))\n        goto err;\n    if (!BN_mod(&(mont->RR), &(mont->RR), &(mont->N), ctx))\n        goto err;\n    ret = 1;\n err:\n    BN_CTX_end(ctx);\n    return ret;\n}']