crypto/rsa/rsa_chk.c:86: error: INTEGER_OVERFLOW_L2
  (0 - [0, max(1, `key->p->top`)]):unsigned64 by call to `BN_mul`.
Showing all 17 steps of the trace


crypto/rsa/rsa_chk.c:19:1: Parameter `key->p->top`
17.   }
18.   
19. > int RSA_check_key_ex(const RSA *key, BN_GENCB *cb)
20.   {
21.       BIGNUM *i, *j, *k, *l, *m;

crypto/rsa/rsa_chk.c:65:9: Call
63.   
64.       /* p prime? */
65.       if (BN_is_prime_ex(key->p, BN_prime_checks, NULL, cb) != 1) {
              ^
66.           ret = 0;
67.           RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_P_NOT_PRIME);

crypto/bn/bn_prime.c:145:1: Parameter `a->top`
  143.   }
  144.   
  145. > int BN_is_prime_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed,
  146.                      BN_GENCB *cb)
  147.   {

crypto/bn/bn_prime.c:148:12: Call
  146.                      BN_GENCB *cb)
  147.   {
  148.       return BN_is_prime_fasttest_ex(a, checks, ctx_passed, 0, cb);
                    ^
  149.   }
  150.   

crypto/bn/bn_prime.c:151:1: Parameter `a->top`
    149.   }
    150.   
    151. > int BN_is_prime_fasttest_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed,
    152.                               int do_trial_division, BN_GENCB *cb)
    153.   {

crypto/bn/bn_prime.c:161:9: Call
    159.   
    160.       /* Take care of the really small primes 2 & 3 */
    161.       if (BN_is_word(a, 2) || BN_is_word(a, 3))
                   ^
    162.           return 1;
    163.   

crypto/bn/bn_lib.c:855:1: Parameter `a->top`
      853.   }
      854.   
      855. > int BN_is_word(const BIGNUM *a, const BN_ULONG w)
      856.   {
      857.       return BN_abs_is_word(a, w) && (!w || !a->neg);

crypto/bn/bn_lib.c:857:12: Call
      855.   int BN_is_word(const BIGNUM *a, const BN_ULONG w)
      856.   {
      857.       return BN_abs_is_word(a, w) && (!w || !a->neg);
                        ^
      858.   }
      859.   

crypto/bn/bn_lib.c:840:1: Parameter `a->top`
        838.   }
        839.   
        840. > int BN_abs_is_word(const BIGNUM *a, const BN_ULONG w)
        841.   {
        842.       return ((a->top == 1) && (a->d[0] == w)) || ((w == 0) && (a->top == 0));

crypto/rsa/rsa_chk.c:86:10: Call
84.   
85.       /* n = p*q * r_3...r_i? */
86.       if (!BN_mul(i, key->p, key->q, ctx)) {
               ^
87.           ret = -1;
88.           goto err;

crypto/bn/bn_mul.c:497:1: Parameter `a->top`
  495.   #endif                          /* BN_RECURSION */
  496.   
  497. > int BN_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)
  498.   {
  499.       int ret = 0;

crypto/bn/bn_mul.c:514:5: Assignment
  512.       bn_check_top(r);
  513.   
  514.       al = a->top;
             ^
  515.       bl = b->top;
  516.   

crypto/bn/bn_mul.c:561:21: Call
  559.                */
  560.               if (i >= 0) {
  561.                   j = BN_num_bits_word((BN_ULONG)al);
                             ^
  562.               }
  563.               if (i == -1) {

crypto/bn/bn_lib.c:89:1: <RHS trace>
87.   }
88.   
89. > int BN_num_bits_word(BN_ULONG l)
90.   {
91.       BN_ULONG x, mask;

crypto/bn/bn_lib.c:89:1: Parameter `l`
    87.   }
    88.   
    89. > int BN_num_bits_word(BN_ULONG l)
    90.   {
    91.       BN_ULONG x, mask;

crypto/bn/bn_lib.c:95:5: Assignment
    93.   
    94.   #if BN_BITS2 > 32
    95.       x = l >> 32;
              ^
    96.       mask = (0 - x) & BN_MASK2;
    97.       mask = (0 - (mask >> (BN_BITS2 - 1)));

crypto/bn/bn_lib.c:96:5: Binary operation: (0 - [0, max(1, key->p->top)]):unsigned64 by call to `BN_mul` 
    94.   #if BN_BITS2 > 32
    95.       x = l >> 32;
    96.       mask = (0 - x) & BN_MASK2;
              ^
    97.       mask = (0 - (mask >> (BN_BITS2 - 1)));
    98.       bits += 32 & mask;

int BN_num_bits_word(BN_ULONG l)
{
    BN_ULONG x, mask;
    int bits = (l != 0);
#if BN_BITS2 > 32
    x = l >> 32;
    mask = (0 - x) & BN_MASK2;
    mask = (0 - (mask >> (BN_BITS2 - 1)));
    bits += 32 & mask;
    l ^= (x ^ l) & mask;
#endif
    x = l >> 16;
    mask = (0 - x) & BN_MASK2;
    mask = (0 - (mask >> (BN_BITS2 - 1)));
    bits += 16 & mask;
    l ^= (x ^ l) & mask;
    x = l >> 8;
    mask = (0 - x) & BN_MASK2;
    mask = (0 - (mask >> (BN_BITS2 - 1)));
    bits += 8 & mask;
    l ^= (x ^ l) & mask;
    x = l >> 4;
    mask = (0 - x) & BN_MASK2;
    mask = (0 - (mask >> (BN_BITS2 - 1)));
    bits += 4 & mask;
    l ^= (x ^ l) & mask;
    x = l >> 2;
    mask = (0 - x) & BN_MASK2;
    mask = (0 - (mask >> (BN_BITS2 - 1)));
    bits += 2 & mask;
    l ^= (x ^ l) & mask;
    x = l >> 1;
    mask = (0 - x) & BN_MASK2;
    mask = (0 - (mask >> (BN_BITS2 - 1)));
    bits += 1 & mask;
    return bits;
}
['int RSA_check_key_ex(const RSA *key, BN_GENCB *cb)\n{\n    BIGNUM *i, *j, *k, *l, *m;\n    BN_CTX *ctx;\n    int ret = 1, ex_primes = 0, idx;\n    RSA_PRIME_INFO *pinfo;\n    if (key->p == NULL || key->q == NULL || key->n == NULL\n            || key->e == NULL || key->d == NULL) {\n        RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_VALUE_MISSING);\n        return 0;\n    }\n    if (key->version == RSA_ASN1_VERSION_MULTI) {\n        ex_primes = sk_RSA_PRIME_INFO_num(key->prime_infos);\n        if (ex_primes <= 0\n                || (ex_primes + 2) > rsa_multip_cap(BN_num_bits(key->n))) {\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_INVALID_MULTI_PRIME_KEY);\n            return 0;\n        }\n    }\n    i = BN_new();\n    j = BN_new();\n    k = BN_new();\n    l = BN_new();\n    m = BN_new();\n    ctx = BN_CTX_new();\n    if (i == NULL || j == NULL || k == NULL || l == NULL\n            || m == NULL || ctx == NULL) {\n        ret = -1;\n        RSAerr(RSA_F_RSA_CHECK_KEY_EX, ERR_R_MALLOC_FAILURE);\n        goto err;\n    }\n    if (BN_is_one(key->e)) {\n        ret = 0;\n        RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_BAD_E_VALUE);\n    }\n    if (!BN_is_odd(key->e)) {\n        ret = 0;\n        RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_BAD_E_VALUE);\n    }\n    if (BN_is_prime_ex(key->p, BN_prime_checks, NULL, cb) != 1) {\n        ret = 0;\n        RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_P_NOT_PRIME);\n    }\n    if (BN_is_prime_ex(key->q, BN_prime_checks, NULL, cb) != 1) {\n        ret = 0;\n        RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_Q_NOT_PRIME);\n    }\n    for (idx = 0; idx < ex_primes; idx++) {\n        pinfo = sk_RSA_PRIME_INFO_value(key->prime_infos, idx);\n        if (BN_is_prime_ex(pinfo->r, BN_prime_checks, NULL, cb) != 1) {\n            ret = 0;\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_MP_R_NOT_PRIME);\n        }\n    }\n    if (!BN_mul(i, key->p, key->q, ctx)) {\n        ret = -1;\n        goto err;\n    }\n    for (idx = 0; idx < ex_primes; idx++) {\n        pinfo = sk_RSA_PRIME_INFO_value(key->prime_infos, idx);\n        if (!BN_mul(i, i, pinfo->r, ctx)) {\n            ret = -1;\n            goto err;\n        }\n    }\n    if (BN_cmp(i, key->n) != 0) {\n        ret = 0;\n        if (ex_primes)\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX,\n                   RSA_R_N_DOES_NOT_EQUAL_PRODUCT_OF_PRIMES);\n        else\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_N_DOES_NOT_EQUAL_P_Q);\n    }\n    if (!BN_sub(i, key->p, BN_value_one())) {\n        ret = -1;\n        goto err;\n    }\n    if (!BN_sub(j, key->q, BN_value_one())) {\n        ret = -1;\n        goto err;\n    }\n    if (!BN_mul(l, i, j, ctx)) {\n        ret = -1;\n        goto err;\n    }\n    if (!BN_gcd(m, i, j, ctx)) {\n        ret = -1;\n        goto err;\n    }\n    for (idx = 0; idx < ex_primes; idx++) {\n        pinfo = sk_RSA_PRIME_INFO_value(key->prime_infos, idx);\n        if (!BN_sub(k, pinfo->r, BN_value_one())) {\n            ret = -1;\n            goto err;\n        }\n        if (!BN_mul(l, l, k, ctx)) {\n            ret = -1;\n            goto err;\n        }\n        if (!BN_gcd(m, m, k, ctx)) {\n            ret = -1;\n            goto err;\n        }\n    }\n    if (!BN_div(k, NULL, l, m, ctx)) {\n        ret = -1;\n        goto err;\n    }\n    if (!BN_mod_mul(i, key->d, key->e, k, ctx)) {\n        ret = -1;\n        goto err;\n    }\n    if (!BN_is_one(i)) {\n        ret = 0;\n        RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_D_E_NOT_CONGRUENT_TO_1);\n    }\n    if (key->dmp1 != NULL && key->dmq1 != NULL && key->iqmp != NULL) {\n        if (!BN_sub(i, key->p, BN_value_one())) {\n            ret = -1;\n            goto err;\n        }\n        if (!BN_mod(j, key->d, i, ctx)) {\n            ret = -1;\n            goto err;\n        }\n        if (BN_cmp(j, key->dmp1) != 0) {\n            ret = 0;\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_DMP1_NOT_CONGRUENT_TO_D);\n        }\n        if (!BN_sub(i, key->q, BN_value_one())) {\n            ret = -1;\n            goto err;\n        }\n        if (!BN_mod(j, key->d, i, ctx)) {\n            ret = -1;\n            goto err;\n        }\n        if (BN_cmp(j, key->dmq1) != 0) {\n            ret = 0;\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_DMQ1_NOT_CONGRUENT_TO_D);\n        }\n        if (!BN_mod_inverse(i, key->q, key->p, ctx)) {\n            ret = -1;\n            goto err;\n        }\n        if (BN_cmp(i, key->iqmp) != 0) {\n            ret = 0;\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_IQMP_NOT_INVERSE_OF_Q);\n        }\n    }\n    for (idx = 0; idx < ex_primes; idx++) {\n        pinfo = sk_RSA_PRIME_INFO_value(key->prime_infos, idx);\n        if (!BN_sub(i, pinfo->r, BN_value_one())) {\n            ret = -1;\n            goto err;\n        }\n        if (!BN_mod(j, key->d, i, ctx)) {\n            ret = -1;\n            goto err;\n        }\n        if (BN_cmp(j, pinfo->d) != 0) {\n            ret = 0;\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_MP_EXPONENT_NOT_CONGRUENT_TO_D);\n        }\n        if (!BN_mod_inverse(i, pinfo->pp, pinfo->r, ctx)) {\n            ret = -1;\n            goto err;\n        }\n        if (BN_cmp(i, pinfo->t) != 0) {\n            ret = 0;\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_MP_COEFFICIENT_NOT_INVERSE_OF_R);\n        }\n    }\n err:\n    BN_free(i);\n    BN_free(j);\n    BN_free(k);\n    BN_free(l);\n    BN_free(m);\n    BN_CTX_free(ctx);\n    return ret;\n}', 'int BN_is_prime_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed,\n                   BN_GENCB *cb)\n{\n    return BN_is_prime_fasttest_ex(a, checks, ctx_passed, 0, cb);\n}', 'int BN_is_prime_fasttest_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed,\n                            int do_trial_division, BN_GENCB *cb)\n{\n    int i, j, ret = -1;\n    int k;\n    BN_CTX *ctx = NULL;\n    BIGNUM *A1, *A1_odd, *A3, *check;\n    BN_MONT_CTX *mont = NULL;\n    if (BN_is_word(a, 2) || BN_is_word(a, 3))\n        return 1;\n    if (!BN_is_odd(a) || BN_cmp(a, BN_value_one()) <= 0)\n        return 0;\n    if (checks == BN_prime_checks)\n        checks = BN_prime_checks_for_size(BN_num_bits(a));\n    if (do_trial_division) {\n        for (i = 1; i < NUMPRIMES; i++) {\n            BN_ULONG mod = BN_mod_word(a, primes[i]);\n            if (mod == (BN_ULONG)-1)\n                goto err;\n            if (mod == 0)\n                return BN_is_word(a, primes[i]);\n        }\n        if (!BN_GENCB_call(cb, 1, -1))\n            goto err;\n    }\n    if (ctx_passed != NULL)\n        ctx = ctx_passed;\n    else if ((ctx = BN_CTX_new()) == NULL)\n        goto err;\n    BN_CTX_start(ctx);\n    A1 = BN_CTX_get(ctx);\n    A3 = BN_CTX_get(ctx);\n    A1_odd = BN_CTX_get(ctx);\n    check = BN_CTX_get(ctx);\n    if (check == NULL)\n        goto err;\n    if (!BN_copy(A1, a) || !BN_sub_word(A1, 1))\n        goto err;\n    if (!BN_copy(A3, a) || !BN_sub_word(A3, 3))\n        goto err;\n    k = 1;\n    while (!BN_is_bit_set(A1, k))\n        k++;\n    if (!BN_rshift(A1_odd, A1, k))\n        goto err;\n    mont = BN_MONT_CTX_new();\n    if (mont == NULL)\n        goto err;\n    if (!BN_MONT_CTX_set(mont, a, ctx))\n        goto err;\n    for (i = 0; i < checks; i++) {\n        if (!BN_priv_rand_range(check, A3) || !BN_add_word(check, 2))\n            goto err;\n        j = witness(check, a, A1, A1_odd, k, ctx, mont);\n        if (j == -1)\n            goto err;\n        if (j) {\n            ret = 0;\n            goto err;\n        }\n        if (!BN_GENCB_call(cb, 1, i))\n            goto err;\n    }\n    ret = 1;\n err:\n    if (ctx != NULL) {\n        BN_CTX_end(ctx);\n        if (ctx_passed == NULL)\n            BN_CTX_free(ctx);\n    }\n    BN_MONT_CTX_free(mont);\n    return ret;\n}', 'int BN_is_word(const BIGNUM *a, const BN_ULONG w)\n{\n    return BN_abs_is_word(a, w) && (!w || !a->neg);\n}', 'int BN_abs_is_word(const BIGNUM *a, const BN_ULONG w)\n{\n    return ((a->top == 1) && (a->d[0] == w)) || ((w == 0) && (a->top == 0));\n}', 'int BN_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)\n{\n    int ret = 0;\n    int top, al, bl;\n    BIGNUM *rr;\n#if defined(BN_MUL_COMBA) || defined(BN_RECURSION)\n    int i;\n#endif\n#ifdef BN_RECURSION\n    BIGNUM *t = NULL;\n    int j = 0, k;\n#endif\n    bn_check_top(a);\n    bn_check_top(b);\n    bn_check_top(r);\n    al = a->top;\n    bl = b->top;\n    if ((al == 0) || (bl == 0)) {\n        BN_zero(r);\n        return 1;\n    }\n    top = al + bl;\n    BN_CTX_start(ctx);\n    if ((r == a) || (r == b)) {\n        if ((rr = BN_CTX_get(ctx)) == NULL)\n            goto err;\n    } else\n        rr = r;\n#if defined(BN_MUL_COMBA) || defined(BN_RECURSION)\n    i = al - bl;\n#endif\n#ifdef BN_MUL_COMBA\n    if (i == 0) {\n# if 0\n        if (al == 4) {\n            if (bn_wexpand(rr, 8) == NULL)\n                goto err;\n            rr->top = 8;\n            bn_mul_comba4(rr->d, a->d, b->d);\n            goto end;\n        }\n# endif\n        if (al == 8) {\n            if (bn_wexpand(rr, 16) == NULL)\n                goto err;\n            rr->top = 16;\n            bn_mul_comba8(rr->d, a->d, b->d);\n            goto end;\n        }\n    }\n#endif\n#ifdef BN_RECURSION\n    if ((al >= BN_MULL_SIZE_NORMAL) && (bl >= BN_MULL_SIZE_NORMAL)) {\n        if (i >= -1 && i <= 1) {\n            if (i >= 0) {\n                j = BN_num_bits_word((BN_ULONG)al);\n            }\n            if (i == -1) {\n                j = BN_num_bits_word((BN_ULONG)bl);\n            }\n            j = 1 << (j - 1);\n            assert(j <= al || j <= bl);\n            k = j + j;\n            t = BN_CTX_get(ctx);\n            if (t == NULL)\n                goto err;\n            if (al > j || bl > j) {\n                if (bn_wexpand(t, k * 4) == NULL)\n                    goto err;\n                if (bn_wexpand(rr, k * 4) == NULL)\n                    goto err;\n                bn_mul_part_recursive(rr->d, a->d, b->d,\n                                      j, al - j, bl - j, t->d);\n            } else {\n                if (bn_wexpand(t, k * 2) == NULL)\n                    goto err;\n                if (bn_wexpand(rr, k * 2) == NULL)\n                    goto err;\n                bn_mul_recursive(rr->d, a->d, b->d, j, al - j, bl - j, t->d);\n            }\n            rr->top = top;\n            goto end;\n        }\n    }\n#endif\n    if (bn_wexpand(rr, top) == NULL)\n        goto err;\n    rr->top = top;\n    bn_mul_normal(rr->d, a->d, al, b->d, bl);\n#if defined(BN_MUL_COMBA) || defined(BN_RECURSION)\n end:\n#endif\n    rr->neg = a->neg ^ b->neg;\n    bn_correct_top(rr);\n    if (r != rr && BN_copy(r, rr) == NULL)\n        goto err;\n    ret = 1;\n err:\n    bn_check_top(r);\n    BN_CTX_end(ctx);\n    return ret;\n}', 'int BN_num_bits_word(BN_ULONG l)\n{\n    BN_ULONG x, mask;\n    int bits = (l != 0);\n#if BN_BITS2 > 32\n    x = l >> 32;\n    mask = (0 - x) & BN_MASK2;\n    mask = (0 - (mask >> (BN_BITS2 - 1)));\n    bits += 32 & mask;\n    l ^= (x ^ l) & mask;\n#endif\n    x = l >> 16;\n    mask = (0 - x) & BN_MASK2;\n    mask = (0 - (mask >> (BN_BITS2 - 1)));\n    bits += 16 & mask;\n    l ^= (x ^ l) & mask;\n    x = l >> 8;\n    mask = (0 - x) & BN_MASK2;\n    mask = (0 - (mask >> (BN_BITS2 - 1)));\n    bits += 8 & mask;\n    l ^= (x ^ l) & mask;\n    x = l >> 4;\n    mask = (0 - x) & BN_MASK2;\n    mask = (0 - (mask >> (BN_BITS2 - 1)));\n    bits += 4 & mask;\n    l ^= (x ^ l) & mask;\n    x = l >> 2;\n    mask = (0 - x) & BN_MASK2;\n    mask = (0 - (mask >> (BN_BITS2 - 1)));\n    bits += 2 & mask;\n    l ^= (x ^ l) & mask;\n    x = l >> 1;\n    mask = (0 - x) & BN_MASK2;\n    mask = (0 - (mask >> (BN_BITS2 - 1)));\n    bits += 1 & mask;\n    return bits;\n}']