test/bntest.c:1427: error: INTEGER_OVERFLOW_L2
  ([0, +oo] - 1):unsigned32 by call to `BN_mod_sqrt`.
Showing all 16 steps of the trace


test/bntest.c:1427:10: Call
1425.   
1426.       /* There are two possible answers. */
1427.       if (!TEST_true(BN_mod_sqrt(ret, a, p, ctx))
                 ^
1428.               || !TEST_true(BN_sub(ret2, p, ret)))
1429.           goto err;

crypto/bn/bn_sqrt.c:13:1: Parameter `ctx->stack.depth`
  11.   #include "bn_lcl.h"
  12.   
  13. > BIGNUM *BN_mod_sqrt(BIGNUM *in, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
  14.   /*
  15.    * Returns 'ret' such that ret^2 == a (mod p), using the Tonelli/Shanks

crypto/bn/bn_sqrt.c:59:5: Call
  57.       }
  58.   
  59.       BN_CTX_start(ctx);
            ^
  60.       A = BN_CTX_get(ctx);
  61.       b = BN_CTX_get(ctx);

crypto/bn/bn_ctx.c:181:1: Parameter `ctx->stack.depth`
    179.   }
    180.   
    181. > void BN_CTX_start(BN_CTX *ctx)
    182.   {
    183.       CTXDBG_ENTRY("BN_CTX_start", ctx);

crypto/bn/bn_sqrt.c:75:10: Call
  73.   
  74.       /* A = a mod p */
  75.       if (!BN_nnmod(A, a, p, ctx))
                 ^
  76.           goto end;
  77.   

crypto/bn/bn_mod.c:13:1: Parameter `ctx->stack.depth`
    11.   #include "bn_lcl.h"
    12.   
    13. > int BN_nnmod(BIGNUM *r, const BIGNUM *m, const BIGNUM *d, BN_CTX *ctx)
    14.   {
    15.       /*

crypto/bn/bn_mod.c:20:11: Call
    18.        */
    19.   
    20.       if (!(BN_mod(r, m, d, ctx)))
                    ^
    21.           return 0;
    22.       if (!r->neg)

crypto/bn/bn_div.c:140:1: Parameter `ctx->stack.depth`
      138.    * If 'dv' or 'rm' is NULL, the respective value is not returned.
      139.    */
      140. > int BN_div(BIGNUM *dv, BIGNUM *rm, const BIGNUM *num, const BIGNUM *divisor,
      141.              BN_CTX *ctx)
      142.   {

crypto/bn/bn_div.c:193:5: Call
      191.       }
      192.   
      193.       BN_CTX_start(ctx);
                 ^
      194.       res = (dv == NULL) ? BN_CTX_get(ctx) : dv;
      195.       tmp = BN_CTX_get(ctx);

crypto/bn/bn_ctx.c:181:1: Parameter `ctx->stack.depth`
        179.   }
        180.   
        181. > void BN_CTX_start(BN_CTX *ctx)
        182.   {
        183.       CTXDBG_ENTRY("BN_CTX_start", ctx);

crypto/bn/bn_div.c:413:5: Call
      411.       if (no_branch)
      412.           bn_correct_top(res);
      413.       BN_CTX_end(ctx);
                 ^
      414.       return 1;
      415.    err:

crypto/bn/bn_ctx.c:195:1: Parameter `ctx->stack.depth`
        193.   }
        194.   
        195. > void BN_CTX_end(BN_CTX *ctx)
        196.   {
        197.       CTXDBG_ENTRY("BN_CTX_end", ctx);

crypto/bn/bn_ctx.c:201:27: Call
        199.           ctx->err_stack--;
        200.       else {
        201.           unsigned int fp = BN_STACK_pop(&ctx->stack);
                                         ^
        202.           /* Does this stack frame have anything to release? */
        203.           if (fp < ctx->used)

crypto/bn/bn_ctx.c:271:1: <LHS trace>
269.   }
270.   
271. > static unsigned int BN_STACK_pop(BN_STACK *st)
272.   {
273.       return st->indexes[--(st->depth)];

crypto/bn/bn_ctx.c:271:1: Parameter `st->depth`
          269.   }
          270.   
          271. > static unsigned int BN_STACK_pop(BN_STACK *st)
          272.   {
          273.       return st->indexes[--(st->depth)];

crypto/bn/bn_ctx.c:273:12: Binary operation: ([0, +oo] - 1):unsigned32 by call to `BN_mod_sqrt` 
          271.   static unsigned int BN_STACK_pop(BN_STACK *st)
          272.   {
          273.       return st->indexes[--(st->depth)];
                            ^
          274.   }
          275.   

static unsigned int BN_STACK_pop(BN_STACK *st)
{
    return st->indexes[--(st->depth)];
}
['static int file_modsqrt(STANZA *s)\n{\n    BIGNUM *a = NULL, *p = NULL, *mod_sqrt = NULL, *ret = NULL, *ret2 = NULL;\n    int st = 0;\n    if (!TEST_ptr(a = getBN(s, "A"))\n            || !TEST_ptr(p = getBN(s, "P"))\n            || !TEST_ptr(mod_sqrt = getBN(s, "ModSqrt"))\n            || !TEST_ptr(ret = BN_new())\n            || !TEST_ptr(ret2 = BN_new()))\n        goto err;\n    if (!TEST_true(BN_mod_sqrt(ret, a, p, ctx))\n            || !TEST_true(BN_sub(ret2, p, ret)))\n        goto err;\n    if (BN_cmp(ret2, mod_sqrt) != 0\n            && !equalBN("sqrt(A) (mod P)", mod_sqrt, ret))\n        goto err;\n    st = 1;\nerr:\n    BN_free(a);\n    BN_free(p);\n    BN_free(mod_sqrt);\n    BN_free(ret);\n    BN_free(ret2);\n    return st;\n}', 'BIGNUM *BN_mod_sqrt(BIGNUM *in, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)\n{\n    BIGNUM *ret = in;\n    int err = 1;\n    int r;\n    BIGNUM *A, *b, *q, *t, *x, *y;\n    int e, i, j;\n    if (!BN_is_odd(p) || BN_abs_is_word(p, 1)) {\n        if (BN_abs_is_word(p, 2)) {\n            if (ret == NULL)\n                ret = BN_new();\n            if (ret == NULL)\n                goto end;\n            if (!BN_set_word(ret, BN_is_bit_set(a, 0))) {\n                if (ret != in)\n                    BN_free(ret);\n                return NULL;\n            }\n            bn_check_top(ret);\n            return ret;\n        }\n        BNerr(BN_F_BN_MOD_SQRT, BN_R_P_IS_NOT_PRIME);\n        return (NULL);\n    }\n    if (BN_is_zero(a) || BN_is_one(a)) {\n        if (ret == NULL)\n            ret = BN_new();\n        if (ret == NULL)\n            goto end;\n        if (!BN_set_word(ret, BN_is_one(a))) {\n            if (ret != in)\n                BN_free(ret);\n            return NULL;\n        }\n        bn_check_top(ret);\n        return ret;\n    }\n    BN_CTX_start(ctx);\n    A = BN_CTX_get(ctx);\n    b = BN_CTX_get(ctx);\n    q = BN_CTX_get(ctx);\n    t = BN_CTX_get(ctx);\n    x = BN_CTX_get(ctx);\n    y = BN_CTX_get(ctx);\n    if (y == NULL)\n        goto end;\n    if (ret == NULL)\n        ret = BN_new();\n    if (ret == NULL)\n        goto end;\n    if (!BN_nnmod(A, a, p, ctx))\n        goto end;\n    e = 1;\n    while (!BN_is_bit_set(p, e))\n        e++;\n    if (e == 1) {\n        if (!BN_rshift(q, p, 2))\n            goto end;\n        q->neg = 0;\n        if (!BN_add_word(q, 1))\n            goto end;\n        if (!BN_mod_exp(ret, A, q, p, ctx))\n            goto end;\n        err = 0;\n        goto vrfy;\n    }\n    if (e == 2) {\n        if (!BN_mod_lshift1_quick(t, A, p))\n            goto end;\n        if (!BN_rshift(q, p, 3))\n            goto end;\n        q->neg = 0;\n        if (!BN_mod_exp(b, t, q, p, ctx))\n            goto end;\n        if (!BN_mod_sqr(y, b, p, ctx))\n            goto end;\n        if (!BN_mod_mul(t, t, y, p, ctx))\n            goto end;\n        if (!BN_sub_word(t, 1))\n            goto end;\n        if (!BN_mod_mul(x, A, b, p, ctx))\n            goto end;\n        if (!BN_mod_mul(x, x, t, p, ctx))\n            goto end;\n        if (!BN_copy(ret, x))\n            goto end;\n        err = 0;\n        goto vrfy;\n    }\n    if (!BN_copy(q, p))\n        goto end;\n    q->neg = 0;\n    i = 2;\n    do {\n        if (i < 22) {\n            if (!BN_set_word(y, i))\n                goto end;\n        } else {\n            if (!BN_rand(y, BN_num_bits(p), 0, 0))\n                goto end;\n            if (BN_ucmp(y, p) >= 0) {\n                if (!(p->neg ? BN_add : BN_sub) (y, y, p))\n                    goto end;\n            }\n            if (BN_is_zero(y))\n                if (!BN_set_word(y, i))\n                    goto end;\n        }\n        r = BN_kronecker(y, q, ctx);\n        if (r < -1)\n            goto end;\n        if (r == 0) {\n            BNerr(BN_F_BN_MOD_SQRT, BN_R_P_IS_NOT_PRIME);\n            goto end;\n        }\n    }\n    while (r == 1 && ++i < 82);\n    if (r != -1) {\n        BNerr(BN_F_BN_MOD_SQRT, BN_R_TOO_MANY_ITERATIONS);\n        goto end;\n    }\n    if (!BN_rshift(q, q, e))\n        goto end;\n    if (!BN_mod_exp(y, y, q, p, ctx))\n        goto end;\n    if (BN_is_one(y)) {\n        BNerr(BN_F_BN_MOD_SQRT, BN_R_P_IS_NOT_PRIME);\n        goto end;\n    }\n    if (!BN_rshift1(t, q))\n        goto end;\n    if (BN_is_zero(t)) {\n        if (!BN_nnmod(t, A, p, ctx))\n            goto end;\n        if (BN_is_zero(t)) {\n            BN_zero(ret);\n            err = 0;\n            goto end;\n        } else if (!BN_one(x))\n            goto end;\n    } else {\n        if (!BN_mod_exp(x, A, t, p, ctx))\n            goto end;\n        if (BN_is_zero(x)) {\n            BN_zero(ret);\n            err = 0;\n            goto end;\n        }\n    }\n    if (!BN_mod_sqr(b, x, p, ctx))\n        goto end;\n    if (!BN_mod_mul(b, b, A, p, ctx))\n        goto end;\n    if (!BN_mod_mul(x, x, A, p, ctx))\n        goto end;\n    while (1) {\n        if (BN_is_one(b)) {\n            if (!BN_copy(ret, x))\n                goto end;\n            err = 0;\n            goto vrfy;\n        }\n        i = 1;\n        if (!BN_mod_sqr(t, b, p, ctx))\n            goto end;\n        while (!BN_is_one(t)) {\n            i++;\n            if (i == e) {\n                BNerr(BN_F_BN_MOD_SQRT, BN_R_NOT_A_SQUARE);\n                goto end;\n            }\n            if (!BN_mod_mul(t, t, t, p, ctx))\n                goto end;\n        }\n        if (!BN_copy(t, y))\n            goto end;\n        for (j = e - i - 1; j > 0; j--) {\n            if (!BN_mod_sqr(t, t, p, ctx))\n                goto end;\n        }\n        if (!BN_mod_mul(y, t, t, p, ctx))\n            goto end;\n        if (!BN_mod_mul(x, x, t, p, ctx))\n            goto end;\n        if (!BN_mod_mul(b, b, y, p, ctx))\n            goto end;\n        e = i;\n    }\n vrfy:\n    if (!err) {\n        if (!BN_mod_sqr(x, ret, p, ctx))\n            err = 1;\n        if (!err && 0 != BN_cmp(x, A)) {\n            BNerr(BN_F_BN_MOD_SQRT, BN_R_NOT_A_SQUARE);\n            err = 1;\n        }\n    }\n end:\n    if (err) {\n        if (ret != in)\n            BN_clear_free(ret);\n        ret = NULL;\n    }\n    BN_CTX_end(ctx);\n    bn_check_top(ret);\n    return ret;\n}', 'void BN_CTX_start(BN_CTX *ctx)\n{\n    CTXDBG_ENTRY("BN_CTX_start", ctx);\n    if (ctx->err_stack || ctx->too_many)\n        ctx->err_stack++;\n    else if (!BN_STACK_push(&ctx->stack, ctx->used)) {\n        BNerr(BN_F_BN_CTX_START, BN_R_TOO_MANY_TEMPORARY_VARIABLES);\n        ctx->err_stack++;\n    }\n    CTXDBG_EXIT(ctx);\n}', 'int BN_nnmod(BIGNUM *r, const BIGNUM *m, const BIGNUM *d, BN_CTX *ctx)\n{\n    if (!(BN_mod(r, m, d, ctx)))\n        return 0;\n    if (!r->neg)\n        return 1;\n    return (d->neg ? BN_sub : BN_add) (r, r, d);\n}', 'int BN_div(BIGNUM *dv, BIGNUM *rm, const BIGNUM *num, const BIGNUM *divisor,\n           BN_CTX *ctx)\n{\n    int norm_shift, i, loop;\n    BIGNUM *tmp, wnum, *snum, *sdiv, *res;\n    BN_ULONG *resp, *wnump;\n    BN_ULONG d0, d1;\n    int num_n, div_n;\n    int no_branch = 0;\n    if ((num->top > 0 && num->d[num->top - 1] == 0) ||\n        (divisor->top > 0 && divisor->d[divisor->top - 1] == 0)) {\n        BNerr(BN_F_BN_DIV, BN_R_NOT_INITIALIZED);\n        return 0;\n    }\n    bn_check_top(num);\n    bn_check_top(divisor);\n    if ((BN_get_flags(num, BN_FLG_CONSTTIME) != 0)\n        || (BN_get_flags(divisor, BN_FLG_CONSTTIME) != 0)) {\n        no_branch = 1;\n    }\n    bn_check_top(dv);\n    bn_check_top(rm);\n    if (BN_is_zero(divisor)) {\n        BNerr(BN_F_BN_DIV, BN_R_DIV_BY_ZERO);\n        return (0);\n    }\n    if (!no_branch && BN_ucmp(num, divisor) < 0) {\n        if (rm != NULL) {\n            if (BN_copy(rm, num) == NULL)\n                return (0);\n        }\n        if (dv != NULL)\n            BN_zero(dv);\n        return 1;\n    }\n    BN_CTX_start(ctx);\n    res = (dv == NULL) ? BN_CTX_get(ctx) : dv;\n    tmp = BN_CTX_get(ctx);\n    snum = BN_CTX_get(ctx);\n    sdiv = BN_CTX_get(ctx);\n    if (sdiv == NULL)\n        goto err;\n    norm_shift = BN_BITS2 - ((BN_num_bits(divisor)) % BN_BITS2);\n    if (!(BN_lshift(sdiv, divisor, norm_shift)))\n        goto err;\n    sdiv->neg = 0;\n    norm_shift += BN_BITS2;\n    if (!(BN_lshift(snum, num, norm_shift)))\n        goto err;\n    snum->neg = 0;\n    if (no_branch) {\n        if (snum->top <= sdiv->top + 1) {\n            if (bn_wexpand(snum, sdiv->top + 2) == NULL)\n                goto err;\n            for (i = snum->top; i < sdiv->top + 2; i++)\n                snum->d[i] = 0;\n            snum->top = sdiv->top + 2;\n        } else {\n            if (bn_wexpand(snum, snum->top + 1) == NULL)\n                goto err;\n            snum->d[snum->top] = 0;\n            snum->top++;\n        }\n    }\n    div_n = sdiv->top;\n    num_n = snum->top;\n    loop = num_n - div_n;\n    wnum.neg = 0;\n    wnum.d = &(snum->d[loop]);\n    wnum.top = div_n;\n    wnum.dmax = snum->dmax - loop;\n    d0 = sdiv->d[div_n - 1];\n    d1 = (div_n == 1) ? 0 : sdiv->d[div_n - 2];\n    wnump = &(snum->d[num_n - 1]);\n    if (!bn_wexpand(res, (loop + 1)))\n        goto err;\n    res->neg = (num->neg ^ divisor->neg);\n    res->top = loop - no_branch;\n    resp = &(res->d[loop - 1]);\n    if (!bn_wexpand(tmp, (div_n + 1)))\n        goto err;\n    if (!no_branch) {\n        if (BN_ucmp(&wnum, sdiv) >= 0) {\n            bn_clear_top2max(&wnum);\n            bn_sub_words(wnum.d, wnum.d, sdiv->d, div_n);\n            *resp = 1;\n        } else\n            res->top--;\n    }\n    resp++;\n    if (res->top == 0)\n        res->neg = 0;\n    else\n        resp--;\n    for (i = 0; i < loop - 1; i++, wnump--) {\n        BN_ULONG q, l0;\n# if defined(BN_DIV3W) && !defined(OPENSSL_NO_ASM)\n        BN_ULONG bn_div_3_words(BN_ULONG *, BN_ULONG, BN_ULONG);\n        q = bn_div_3_words(wnump, d1, d0);\n# else\n        BN_ULONG n0, n1, rem = 0;\n        n0 = wnump[0];\n        n1 = wnump[-1];\n        if (n0 == d0)\n            q = BN_MASK2;\n        else {\n#  ifdef BN_LLONG\n            BN_ULLONG t2;\n#   if defined(BN_LLONG) && defined(BN_DIV2W) && !defined(bn_div_words)\n            q = (BN_ULONG)(((((BN_ULLONG) n0) << BN_BITS2) | n1) / d0);\n#   else\n            q = bn_div_words(n0, n1, d0);\n#   endif\n#   ifndef REMAINDER_IS_ALREADY_CALCULATED\n            rem = (n1 - q * d0) & BN_MASK2;\n#   endif\n            t2 = (BN_ULLONG) d1 *q;\n            for (;;) {\n                if (t2 <= ((((BN_ULLONG) rem) << BN_BITS2) | wnump[-2]))\n                    break;\n                q--;\n                rem += d0;\n                if (rem < d0)\n                    break;\n                t2 -= d1;\n            }\n#  else\n            BN_ULONG t2l, t2h;\n            q = bn_div_words(n0, n1, d0);\n#   ifndef REMAINDER_IS_ALREADY_CALCULATED\n            rem = (n1 - q * d0) & BN_MASK2;\n#   endif\n#   if defined(BN_UMULT_LOHI)\n            BN_UMULT_LOHI(t2l, t2h, d1, q);\n#   elif defined(BN_UMULT_HIGH)\n            t2l = d1 * q;\n            t2h = BN_UMULT_HIGH(d1, q);\n#   else\n            {\n                BN_ULONG ql, qh;\n                t2l = LBITS(d1);\n                t2h = HBITS(d1);\n                ql = LBITS(q);\n                qh = HBITS(q);\n                mul64(t2l, t2h, ql, qh);\n            }\n#   endif\n            for (;;) {\n                if ((t2h < rem) || ((t2h == rem) && (t2l <= wnump[-2])))\n                    break;\n                q--;\n                rem += d0;\n                if (rem < d0)\n                    break;\n                if (t2l < d1)\n                    t2h--;\n                t2l -= d1;\n            }\n#  endif\n        }\n# endif\n        l0 = bn_mul_words(tmp->d, sdiv->d, div_n, q);\n        tmp->d[div_n] = l0;\n        wnum.d--;\n        if (bn_sub_words(wnum.d, wnum.d, tmp->d, div_n + 1)) {\n            q--;\n            if (bn_add_words(wnum.d, wnum.d, sdiv->d, div_n))\n                (*wnump)++;\n        }\n        resp--;\n        *resp = q;\n    }\n    bn_correct_top(snum);\n    if (rm != NULL) {\n        int neg = num->neg;\n        BN_rshift(rm, snum, norm_shift);\n        if (!BN_is_zero(rm))\n            rm->neg = neg;\n        bn_check_top(rm);\n    }\n    if (no_branch)\n        bn_correct_top(res);\n    BN_CTX_end(ctx);\n    return 1;\n err:\n    bn_check_top(rm);\n    BN_CTX_end(ctx);\n    return (0);\n}', 'void BN_CTX_end(BN_CTX *ctx)\n{\n    CTXDBG_ENTRY("BN_CTX_end", ctx);\n    if (ctx->err_stack)\n        ctx->err_stack--;\n    else {\n        unsigned int fp = BN_STACK_pop(&ctx->stack);\n        if (fp < ctx->used)\n            BN_POOL_release(&ctx->pool, ctx->used - fp);\n        ctx->used = fp;\n        ctx->too_many = 0;\n    }\n    CTXDBG_EXIT(ctx);\n}', 'static unsigned int BN_STACK_pop(BN_STACK *st)\n{\n    return st->indexes[--(st->depth)];\n}']