crypto/rsa/rsa_chk.c:104: error: INTEGER_OVERFLOW_L2
  ([0, +oo] - 1):unsigned32 by call to `BN_mul`.
Showing all 18 steps of the trace


crypto/rsa/rsa_chk.c:86:9: Call
84.   
85.       /* p prime? */
86.       r = BN_is_prime_ex(key->p, BN_prime_checks, NULL, cb);
              ^
87.       if (r != 1) {
88.           ret = r;

crypto/bn/bn_prime.c:296:1: Parameter `ctx_passed->stack.depth`
  294.   }
  295.   
  296. > int BN_is_prime_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed,
  297.                      BN_GENCB *cb)
  298.   {

crypto/bn/bn_prime.c:299:12: Call
  297.                      BN_GENCB *cb)
  298.   {
  299.       return BN_is_prime_fasttest_ex(a, checks, ctx_passed, 0, cb);
                    ^
  300.   }
  301.   

crypto/bn/bn_prime.c:302:1: Parameter `ctx_passed->stack.depth`
    300.   }
    301.   
    302. > int BN_is_prime_fasttest_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed,
    303.                               int do_trial_division, BN_GENCB *cb)
    304.   {

crypto/rsa/rsa_chk.c:95:9: Call
93.   
94.       /* q prime? */
95.       r = BN_is_prime_ex(key->q, BN_prime_checks, NULL, cb);
              ^
96.       if (r != 1) {
97.           ret = r;

crypto/bn/bn_prime.c:296:1: Parameter `ctx_passed->stack.depth`
  294.   }
  295.   
  296. > int BN_is_prime_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed,
  297.                      BN_GENCB *cb)
  298.   {

crypto/bn/bn_prime.c:299:12: Call
  297.                      BN_GENCB *cb)
  298.   {
  299.       return BN_is_prime_fasttest_ex(a, checks, ctx_passed, 0, cb);
                    ^
  300.   }
  301.   

crypto/bn/bn_prime.c:302:1: Parameter `ctx_passed->stack.depth`
    300.   }
    301.   
    302. > int BN_is_prime_fasttest_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed,
    303.                               int do_trial_division, BN_GENCB *cb)
    304.   {

crypto/rsa/rsa_chk.c:104:9: Call
102.   
103.       /* n = p*q? */
104.       r = BN_mul(i, key->p, key->q, ctx);
               ^
105.       if (!r) {
106.           ret = -1;

crypto/bn/bn_mul.c:883:1: Parameter `ctx->stack.depth`
  881.   #endif                          /* BN_RECURSION */
  882.   
  883. > int BN_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)
  884.   {
  885.       int ret = 0;

crypto/bn/bn_mul.c:909:5: Call
  907.       top = al + bl;
  908.   
  909.       BN_CTX_start(ctx);
             ^
  910.       if ((r == a) || (r == b)) {
  911.           if ((rr = BN_CTX_get(ctx)) == NULL)

crypto/bn/bn_ctx.c:253:1: Parameter `ctx->stack.depth`
    251.   }
    252.   
    253. > void BN_CTX_start(BN_CTX *ctx)
    254.   {
    255.       CTXDBG_ENTRY("BN_CTX_start", ctx);

crypto/bn/bn_mul.c:1033:5: Call
  1031.    err:
  1032.       bn_check_top(r);
  1033.       BN_CTX_end(ctx);
              ^
  1034.       return (ret);
  1035.   }

crypto/bn/bn_ctx.c:267:1: Parameter `ctx->stack.depth`
    265.   }
    266.   
    267. > void BN_CTX_end(BN_CTX *ctx)
    268.   {
    269.       CTXDBG_ENTRY("BN_CTX_end", ctx);

crypto/bn/bn_ctx.c:273:27: Call
    271.           ctx->err_stack--;
    272.       else {
    273.           unsigned int fp = BN_STACK_pop(&ctx->stack);
                                     ^
    274.           /* Does this stack frame have anything to release? */
    275.           if (fp < ctx->used)

crypto/bn/bn_ctx.c:351:1: <LHS trace>
349.   }
350.   
351. > static unsigned int BN_STACK_pop(BN_STACK *st)
352.   {
353.       return st->indexes[--(st->depth)];

crypto/bn/bn_ctx.c:351:1: Parameter `st->depth`
      349.   }
      350.   
      351. > static unsigned int BN_STACK_pop(BN_STACK *st)
      352.   {
      353.       return st->indexes[--(st->depth)];

crypto/bn/bn_ctx.c:353:12: Binary operation: ([0, +oo] - 1):unsigned32 by call to `BN_mul` 
      351.   static unsigned int BN_STACK_pop(BN_STACK *st)
      352.   {
      353.       return st->indexes[--(st->depth)];
                        ^
      354.   }
      355.   

static unsigned int BN_STACK_pop(BN_STACK *st)
{
    return st->indexes[--(st->depth)];
}
['int RSA_check_key_ex(const RSA *key, BN_GENCB *cb)\n{\n    BIGNUM *i, *j, *k, *l, *m;\n    BN_CTX *ctx;\n    int r;\n    int ret = 1;\n    if (!key->p || !key->q || !key->n || !key->e || !key->d) {\n        RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_VALUE_MISSING);\n        return 0;\n    }\n    i = BN_new();\n    j = BN_new();\n    k = BN_new();\n    l = BN_new();\n    m = BN_new();\n    ctx = BN_CTX_new();\n    if (i == NULL || j == NULL || k == NULL || l == NULL ||\n        m == NULL || ctx == NULL) {\n        ret = -1;\n        RSAerr(RSA_F_RSA_CHECK_KEY_EX, ERR_R_MALLOC_FAILURE);\n        goto err;\n    }\n    r = BN_is_prime_ex(key->p, BN_prime_checks, NULL, cb);\n    if (r != 1) {\n        ret = r;\n        if (r != 0)\n            goto err;\n        RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_P_NOT_PRIME);\n    }\n    r = BN_is_prime_ex(key->q, BN_prime_checks, NULL, cb);\n    if (r != 1) {\n        ret = r;\n        if (r != 0)\n            goto err;\n        RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_Q_NOT_PRIME);\n    }\n    r = BN_mul(i, key->p, key->q, ctx);\n    if (!r) {\n        ret = -1;\n        goto err;\n    }\n    if (BN_cmp(i, key->n) != 0) {\n        ret = 0;\n        RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_N_DOES_NOT_EQUAL_P_Q);\n    }\n    r = BN_sub(i, key->p, BN_value_one());\n    if (!r) {\n        ret = -1;\n        goto err;\n    }\n    r = BN_sub(j, key->q, BN_value_one());\n    if (!r) {\n        ret = -1;\n        goto err;\n    }\n    r = BN_mul(l, i, j, ctx);\n    if (!r) {\n        ret = -1;\n        goto err;\n    }\n    r = BN_gcd(m, i, j, ctx);\n    if (!r) {\n        ret = -1;\n        goto err;\n    }\n    r = BN_div(k, NULL, l, m, ctx);\n    if (!r) {\n        ret = -1;\n        goto err;\n    }\n    r = BN_mod_mul(i, key->d, key->e, k, ctx);\n    if (!r) {\n        ret = -1;\n        goto err;\n    }\n    if (!BN_is_one(i)) {\n        ret = 0;\n        RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_D_E_NOT_CONGRUENT_TO_1);\n    }\n    if (key->dmp1 != NULL && key->dmq1 != NULL && key->iqmp != NULL) {\n        r = BN_sub(i, key->p, BN_value_one());\n        if (!r) {\n            ret = -1;\n            goto err;\n        }\n        r = BN_mod(j, key->d, i, ctx);\n        if (!r) {\n            ret = -1;\n            goto err;\n        }\n        if (BN_cmp(j, key->dmp1) != 0) {\n            ret = 0;\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_DMP1_NOT_CONGRUENT_TO_D);\n        }\n        r = BN_sub(i, key->q, BN_value_one());\n        if (!r) {\n            ret = -1;\n            goto err;\n        }\n        r = BN_mod(j, key->d, i, ctx);\n        if (!r) {\n            ret = -1;\n            goto err;\n        }\n        if (BN_cmp(j, key->dmq1) != 0) {\n            ret = 0;\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_DMQ1_NOT_CONGRUENT_TO_D);\n        }\n        if (!BN_mod_inverse(i, key->q, key->p, ctx)) {\n            ret = -1;\n            goto err;\n        }\n        if (BN_cmp(i, key->iqmp) != 0) {\n            ret = 0;\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_IQMP_NOT_INVERSE_OF_Q);\n        }\n    }\n err:\n    if (i != NULL)\n        BN_free(i);\n    if (j != NULL)\n        BN_free(j);\n    if (k != NULL)\n        BN_free(k);\n    if (l != NULL)\n        BN_free(l);\n    if (m != NULL)\n        BN_free(m);\n    if (ctx != NULL)\n        BN_CTX_free(ctx);\n    return (ret);\n}', 'int BN_is_prime_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed,\n                   BN_GENCB *cb)\n{\n    return BN_is_prime_fasttest_ex(a, checks, ctx_passed, 0, cb);\n}', 'int BN_is_prime_fasttest_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed,\n                            int do_trial_division, BN_GENCB *cb)\n{\n    int i, j, ret = -1;\n    int k;\n    BN_CTX *ctx = NULL;\n    BIGNUM *A1, *A1_odd, *check;\n    BN_MONT_CTX *mont = NULL;\n    const BIGNUM *A = NULL;\n    if (BN_cmp(a, BN_value_one()) <= 0)\n        return 0;\n    if (checks == BN_prime_checks)\n        checks = BN_prime_checks_for_size(BN_num_bits(a));\n    if (!BN_is_odd(a))\n        return BN_is_word(a, 2);\n    if (do_trial_division) {\n        for (i = 1; i < NUMPRIMES; i++)\n            if (BN_mod_word(a, primes[i]) == 0)\n                return 0;\n        if (!BN_GENCB_call(cb, 1, -1))\n            goto err;\n    }\n    if (ctx_passed != NULL)\n        ctx = ctx_passed;\n    else if ((ctx = BN_CTX_new()) == NULL)\n        goto err;\n    BN_CTX_start(ctx);\n    if (a->neg) {\n        BIGNUM *t;\n        if ((t = BN_CTX_get(ctx)) == NULL)\n            goto err;\n        BN_copy(t, a);\n        t->neg = 0;\n        A = t;\n    } else\n        A = a;\n    A1 = BN_CTX_get(ctx);\n    A1_odd = BN_CTX_get(ctx);\n    check = BN_CTX_get(ctx);\n    if (check == NULL)\n        goto err;\n    if (!BN_copy(A1, A))\n        goto err;\n    if (!BN_sub_word(A1, 1))\n        goto err;\n    if (BN_is_zero(A1)) {\n        ret = 0;\n        goto err;\n    }\n    k = 1;\n    while (!BN_is_bit_set(A1, k))\n        k++;\n    if (!BN_rshift(A1_odd, A1, k))\n        goto err;\n    mont = BN_MONT_CTX_new();\n    if (mont == NULL)\n        goto err;\n    if (!BN_MONT_CTX_set(mont, A, ctx))\n        goto err;\n    for (i = 0; i < checks; i++) {\n        if (!BN_pseudo_rand_range(check, A1))\n            goto err;\n        if (!BN_add_word(check, 1))\n            goto err;\n        j = witness(check, A, A1, A1_odd, k, ctx, mont);\n        if (j == -1)\n            goto err;\n        if (j) {\n            ret = 0;\n            goto err;\n        }\n        if (!BN_GENCB_call(cb, 1, i))\n            goto err;\n    }\n    ret = 1;\n err:\n    if (ctx != NULL) {\n        BN_CTX_end(ctx);\n        if (ctx_passed == NULL)\n            BN_CTX_free(ctx);\n    }\n    if (mont != NULL)\n        BN_MONT_CTX_free(mont);\n    return (ret);\n}', 'int BN_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)\n{\n    int ret = 0;\n    int top, al, bl;\n    BIGNUM *rr;\n#if defined(BN_MUL_COMBA) || defined(BN_RECURSION)\n    int i;\n#endif\n#ifdef BN_RECURSION\n    BIGNUM *t = NULL;\n    int j = 0, k;\n#endif\n    bn_check_top(a);\n    bn_check_top(b);\n    bn_check_top(r);\n    al = a->top;\n    bl = b->top;\n    if ((al == 0) || (bl == 0)) {\n        BN_zero(r);\n        return (1);\n    }\n    top = al + bl;\n    BN_CTX_start(ctx);\n    if ((r == a) || (r == b)) {\n        if ((rr = BN_CTX_get(ctx)) == NULL)\n            goto err;\n    } else\n        rr = r;\n    rr->neg = a->neg ^ b->neg;\n#if defined(BN_MUL_COMBA) || defined(BN_RECURSION)\n    i = al - bl;\n#endif\n#ifdef BN_MUL_COMBA\n    if (i == 0) {\n# if 0\n        if (al == 4) {\n            if (bn_wexpand(rr, 8) == NULL)\n                goto err;\n            rr->top = 8;\n            bn_mul_comba4(rr->d, a->d, b->d);\n            goto end;\n        }\n# endif\n        if (al == 8) {\n            if (bn_wexpand(rr, 16) == NULL)\n                goto err;\n            rr->top = 16;\n            bn_mul_comba8(rr->d, a->d, b->d);\n            goto end;\n        }\n    }\n#endif\n#ifdef BN_RECURSION\n    if ((al >= BN_MULL_SIZE_NORMAL) && (bl >= BN_MULL_SIZE_NORMAL)) {\n        if (i >= -1 && i <= 1) {\n            if (i >= 0) {\n                j = BN_num_bits_word((BN_ULONG)al);\n            }\n            if (i == -1) {\n                j = BN_num_bits_word((BN_ULONG)bl);\n            }\n            j = 1 << (j - 1);\n            assert(j <= al || j <= bl);\n            k = j + j;\n            t = BN_CTX_get(ctx);\n            if (t == NULL)\n                goto err;\n            if (al > j || bl > j) {\n                if (bn_wexpand(t, k * 4) == NULL)\n                    goto err;\n                if (bn_wexpand(rr, k * 4) == NULL)\n                    goto err;\n                bn_mul_part_recursive(rr->d, a->d, b->d,\n                                      j, al - j, bl - j, t->d);\n            } else {\n                if (bn_wexpand(t, k * 2) == NULL)\n                    goto err;\n                if (bn_wexpand(rr, k * 2) == NULL)\n                    goto err;\n                bn_mul_recursive(rr->d, a->d, b->d, j, al - j, bl - j, t->d);\n            }\n            rr->top = top;\n            goto end;\n        }\n# if 0\n        if (i == 1 && !BN_get_flags(b, BN_FLG_STATIC_DATA)) {\n            BIGNUM *tmp_bn = (BIGNUM *)b;\n            if (bn_wexpand(tmp_bn, al) == NULL)\n                goto err;\n            tmp_bn->d[bl] = 0;\n            bl++;\n            i--;\n        } else if (i == -1 && !BN_get_flags(a, BN_FLG_STATIC_DATA)) {\n            BIGNUM *tmp_bn = (BIGNUM *)a;\n            if (bn_wexpand(tmp_bn, bl) == NULL)\n                goto err;\n            tmp_bn->d[al] = 0;\n            al++;\n            i++;\n        }\n        if (i == 0) {\n            j = BN_num_bits_word((BN_ULONG)al);\n            j = 1 << (j - 1);\n            k = j + j;\n            t = BN_CTX_get(ctx);\n            if (al == j) {\n                if (bn_wexpand(t, k * 2) == NULL)\n                    goto err;\n                if (bn_wexpand(rr, k * 2) == NULL)\n                    goto err;\n                bn_mul_recursive(rr->d, a->d, b->d, al, t->d);\n            } else {\n                if (bn_wexpand(t, k * 4) == NULL)\n                    goto err;\n                if (bn_wexpand(rr, k * 4) == NULL)\n                    goto err;\n                bn_mul_part_recursive(rr->d, a->d, b->d, al - j, j, t->d);\n            }\n            rr->top = top;\n            goto end;\n        }\n# endif\n    }\n#endif\n    if (bn_wexpand(rr, top) == NULL)\n        goto err;\n    rr->top = top;\n    bn_mul_normal(rr->d, a->d, al, b->d, bl);\n#if defined(BN_MUL_COMBA) || defined(BN_RECURSION)\n end:\n#endif\n    bn_correct_top(rr);\n    if (r != rr)\n        BN_copy(r, rr);\n    ret = 1;\n err:\n    bn_check_top(r);\n    BN_CTX_end(ctx);\n    return (ret);\n}', 'void BN_CTX_start(BN_CTX *ctx)\n{\n    CTXDBG_ENTRY("BN_CTX_start", ctx);\n    if (ctx->err_stack || ctx->too_many)\n        ctx->err_stack++;\n    else if (!BN_STACK_push(&ctx->stack, ctx->used)) {\n        BNerr(BN_F_BN_CTX_START, BN_R_TOO_MANY_TEMPORARY_VARIABLES);\n        ctx->err_stack++;\n    }\n    CTXDBG_EXIT(ctx);\n}', 'void BN_CTX_end(BN_CTX *ctx)\n{\n    CTXDBG_ENTRY("BN_CTX_end", ctx);\n    if (ctx->err_stack)\n        ctx->err_stack--;\n    else {\n        unsigned int fp = BN_STACK_pop(&ctx->stack);\n        if (fp < ctx->used)\n            BN_POOL_release(&ctx->pool, ctx->used - fp);\n        ctx->used = fp;\n        ctx->too_many = 0;\n    }\n    CTXDBG_EXIT(ctx);\n}', 'static unsigned int BN_STACK_pop(BN_STACK *st)\n{\n    return st->indexes[--(st->depth)];\n}']