crypto/rsa/rsa_chk.c:157: error: INTEGER_OVERFLOW_L2
  (0 - [0, max(1, `key->e->top`)]):unsigned64 by call to `BN_mod_mul`.
Showing all 19 steps of the trace


crypto/rsa/rsa_chk.c:25:1: Parameter `key->e->top`
23.    *      rsa_sp800_56b_check_keypair.
24.    */
25. > int RSA_check_key_ex(const RSA *key, BN_GENCB *cb)
26.   {
27.   #ifdef FIPS_MODE

crypto/rsa/rsa_chk.c:68:9: Call
66.       }
67.   
68.       if (BN_is_one(key->e)) {
              ^
69.           ret = 0;
70.           RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_BAD_E_VALUE);

crypto/bn/bn_lib.c:869:1: Parameter `a->top`
  867.   }
  868.   
  869. > int BN_is_one(const BIGNUM *a)
  870.   {
  871.       return BN_abs_is_word(a, 1) && !a->neg;

crypto/bn/bn_lib.c:871:12: Call
  869.   int BN_is_one(const BIGNUM *a)
  870.   {
  871.       return BN_abs_is_word(a, 1) && !a->neg;
                    ^
  872.   }
  873.   

crypto/bn/bn_lib.c:859:1: Parameter `a->top`
    857.   }
    858.   
    859. > int BN_abs_is_word(const BIGNUM *a, const BN_ULONG w)
    860.   {
    861.       return ((a->top == 1) && (a->d[0] == w)) || ((w == 0) && (a->top == 0));

crypto/rsa/rsa_chk.c:72:10: Call
70.           RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_BAD_E_VALUE);
71.       }
72.       if (!BN_is_odd(key->e)) {
               ^
73.           ret = 0;
74.           RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_BAD_E_VALUE);

crypto/bn/bn_lib.c:879:1: Parameter `a->top`
  877.   }
  878.   
  879. > int BN_is_odd(const BIGNUM *a)
  880.   {
  881.       return (a->top > 0) && (a->d[0] & 1);

crypto/rsa/rsa_chk.c:157:10: Call
155.           goto err;
156.       }
157.       if (!BN_mod_mul(i, key->d, key->e, k, ctx)) {
                ^
158.           ret = -1;
159.           goto err;

crypto/bn/bn_mod.c:193:1: Parameter `b->top`
  191.   
  192.   /* slow but works */
  193. > int BN_mod_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m,
  194.                  BN_CTX *ctx)
  195.   {

crypto/bn/bn_mod.c:210:14: Call
  208.               goto err;
  209.       } else {
  210.           if (!BN_mul(t, a, b, ctx))
                      ^
  211.               goto err;
  212.       }

crypto/bn/bn_mul.c:497:1: Parameter `b->top`
    495.   #endif                          /* BN_RECURSION */
    496.   
    497. > int BN_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)
    498.   {
    499.       int ret = bn_mul_fixed_top(r, a, b, ctx);

crypto/bn/bn_mul.c:499:15: Call
    497.   int BN_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)
    498.   {
    499.       int ret = bn_mul_fixed_top(r, a, b, ctx);
                         ^
    500.   
    501.       bn_correct_top(r);

crypto/bn/bn_mul.c:507:1: Parameter `b->top`
      505.   }
      506.   
      507. > int bn_mul_fixed_top(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)
      508.   {
      509.       int ret = 0;

crypto/bn/bn_mul.c:525:5: Assignment
      523.   
      524.       al = a->top;
      525.       bl = b->top;
                 ^
      526.   
      527.       if ((al == 0) || (bl == 0)) {

crypto/bn/bn_mul.c:574:21: Call
      572.               }
      573.               if (i == -1) {
      574.                   j = BN_num_bits_word((BN_ULONG)bl);
                                 ^
      575.               }
      576.               j = 1 << (j - 1);

crypto/bn/bn_lib.c:90:1: <RHS trace>
88.   }
89.   
90. > int BN_num_bits_word(BN_ULONG l)
91.   {
92.       BN_ULONG x, mask;

crypto/bn/bn_lib.c:90:1: Parameter `l`
        88.   }
        89.   
        90. > int BN_num_bits_word(BN_ULONG l)
        91.   {
        92.       BN_ULONG x, mask;

crypto/bn/bn_lib.c:96:5: Assignment
        94.   
        95.   #if BN_BITS2 > 32
        96.       x = l >> 32;
                  ^
        97.       mask = (0 - x) & BN_MASK2;
        98.       mask = (0 - (mask >> (BN_BITS2 - 1)));

crypto/bn/bn_lib.c:97:5: Binary operation: (0 - [0, max(1, key->e->top)]):unsigned64 by call to `BN_mod_mul` 
        95.   #if BN_BITS2 > 32
        96.       x = l >> 32;
        97.       mask = (0 - x) & BN_MASK2;
                   ^
        98.       mask = (0 - (mask >> (BN_BITS2 - 1)));
        99.       bits += 32 & mask;

int BN_num_bits_word(BN_ULONG l)
{
    BN_ULONG x, mask;
    int bits = (l != 0);
#if BN_BITS2 > 32
    x = l >> 32;
    mask = (0 - x) & BN_MASK2;
    mask = (0 - (mask >> (BN_BITS2 - 1)));
    bits += 32 & mask;
    l ^= (x ^ l) & mask;
#endif
    x = l >> 16;
    mask = (0 - x) & BN_MASK2;
    mask = (0 - (mask >> (BN_BITS2 - 1)));
    bits += 16 & mask;
    l ^= (x ^ l) & mask;
    x = l >> 8;
    mask = (0 - x) & BN_MASK2;
    mask = (0 - (mask >> (BN_BITS2 - 1)));
    bits += 8 & mask;
    l ^= (x ^ l) & mask;
    x = l >> 4;
    mask = (0 - x) & BN_MASK2;
    mask = (0 - (mask >> (BN_BITS2 - 1)));
    bits += 4 & mask;
    l ^= (x ^ l) & mask;
    x = l >> 2;
    mask = (0 - x) & BN_MASK2;
    mask = (0 - (mask >> (BN_BITS2 - 1)));
    bits += 2 & mask;
    l ^= (x ^ l) & mask;
    x = l >> 1;
    mask = (0 - x) & BN_MASK2;
    mask = (0 - (mask >> (BN_BITS2 - 1)));
    bits += 1 & mask;
    return bits;
}
['int RSA_check_key_ex(const RSA *key, BN_GENCB *cb)\n{\n#ifdef FIPS_MODE\n    if (!(rsa_sp800_56b_check_public(key)\n            && rsa_sp800_56b_check_private(key)\n            && rsa_sp800_56b_check_keypair(key, NULL, -1, RSA_bits(key))\n        return 0;\n#else\n    BIGNUM *i, *j, *k, *l, *m;\n    BN_CTX *ctx;\n    int ret = 1, ex_primes = 0, idx;\n    RSA_PRIME_INFO *pinfo;\n    if (key->p == NULL || key->q == NULL || key->n == NULL\n            || key->e == NULL || key->d == NULL) {\n        RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_VALUE_MISSING);\n        return 0;\n    }\n    if (key->version == RSA_ASN1_VERSION_MULTI) {\n        ex_primes = sk_RSA_PRIME_INFO_num(key->prime_infos);\n        if (ex_primes <= 0\n                || (ex_primes + 2) > rsa_multip_cap(BN_num_bits(key->n))) {\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_INVALID_MULTI_PRIME_KEY);\n            return 0;\n        }\n    }\n    i = BN_new();\n    j = BN_new();\n    k = BN_new();\n    l = BN_new();\n    m = BN_new();\n    ctx = BN_CTX_new();\n    if (i == NULL || j == NULL || k == NULL || l == NULL\n            || m == NULL || ctx == NULL) {\n        ret = -1;\n        RSAerr(RSA_F_RSA_CHECK_KEY_EX, ERR_R_MALLOC_FAILURE);\n        goto err;\n    }\n    if (BN_is_one(key->e)) {\n        ret = 0;\n        RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_BAD_E_VALUE);\n    }\n    if (!BN_is_odd(key->e)) {\n        ret = 0;\n        RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_BAD_E_VALUE);\n    }\n    if (BN_is_prime_ex(key->p, BN_prime_checks, NULL, cb) != 1) {\n        ret = 0;\n        RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_P_NOT_PRIME);\n    }\n    if (BN_is_prime_ex(key->q, BN_prime_checks, NULL, cb) != 1) {\n        ret = 0;\n        RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_Q_NOT_PRIME);\n    }\n    for (idx = 0; idx < ex_primes; idx++) {\n        pinfo = sk_RSA_PRIME_INFO_value(key->prime_infos, idx);\n        if (BN_is_prime_ex(pinfo->r, BN_prime_checks, NULL, cb) != 1) {\n            ret = 0;\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_MP_R_NOT_PRIME);\n        }\n    }\n    if (!BN_mul(i, key->p, key->q, ctx)) {\n        ret = -1;\n        goto err;\n    }\n    for (idx = 0; idx < ex_primes; idx++) {\n        pinfo = sk_RSA_PRIME_INFO_value(key->prime_infos, idx);\n        if (!BN_mul(i, i, pinfo->r, ctx)) {\n            ret = -1;\n            goto err;\n        }\n    }\n    if (BN_cmp(i, key->n) != 0) {\n        ret = 0;\n        if (ex_primes)\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX,\n                   RSA_R_N_DOES_NOT_EQUAL_PRODUCT_OF_PRIMES);\n        else\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_N_DOES_NOT_EQUAL_P_Q);\n    }\n    if (!BN_sub(i, key->p, BN_value_one())) {\n        ret = -1;\n        goto err;\n    }\n    if (!BN_sub(j, key->q, BN_value_one())) {\n        ret = -1;\n        goto err;\n    }\n    if (!BN_mul(l, i, j, ctx)) {\n        ret = -1;\n        goto err;\n    }\n    if (!BN_gcd(m, i, j, ctx)) {\n        ret = -1;\n        goto err;\n    }\n    for (idx = 0; idx < ex_primes; idx++) {\n        pinfo = sk_RSA_PRIME_INFO_value(key->prime_infos, idx);\n        if (!BN_sub(k, pinfo->r, BN_value_one())) {\n            ret = -1;\n            goto err;\n        }\n        if (!BN_mul(l, l, k, ctx)) {\n            ret = -1;\n            goto err;\n        }\n        if (!BN_gcd(m, m, k, ctx)) {\n            ret = -1;\n            goto err;\n        }\n    }\n    if (!BN_div(k, NULL, l, m, ctx)) {\n        ret = -1;\n        goto err;\n    }\n    if (!BN_mod_mul(i, key->d, key->e, k, ctx)) {\n        ret = -1;\n        goto err;\n    }\n    if (!BN_is_one(i)) {\n        ret = 0;\n        RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_D_E_NOT_CONGRUENT_TO_1);\n    }\n    if (key->dmp1 != NULL && key->dmq1 != NULL && key->iqmp != NULL) {\n        if (!BN_sub(i, key->p, BN_value_one())) {\n            ret = -1;\n            goto err;\n        }\n        if (!BN_mod(j, key->d, i, ctx)) {\n            ret = -1;\n            goto err;\n        }\n        if (BN_cmp(j, key->dmp1) != 0) {\n            ret = 0;\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_DMP1_NOT_CONGRUENT_TO_D);\n        }\n        if (!BN_sub(i, key->q, BN_value_one())) {\n            ret = -1;\n            goto err;\n        }\n        if (!BN_mod(j, key->d, i, ctx)) {\n            ret = -1;\n            goto err;\n        }\n        if (BN_cmp(j, key->dmq1) != 0) {\n            ret = 0;\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_DMQ1_NOT_CONGRUENT_TO_D);\n        }\n        if (!BN_mod_inverse(i, key->q, key->p, ctx)) {\n            ret = -1;\n            goto err;\n        }\n        if (BN_cmp(i, key->iqmp) != 0) {\n            ret = 0;\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_IQMP_NOT_INVERSE_OF_Q);\n        }\n    }\n    for (idx = 0; idx < ex_primes; idx++) {\n        pinfo = sk_RSA_PRIME_INFO_value(key->prime_infos, idx);\n        if (!BN_sub(i, pinfo->r, BN_value_one())) {\n            ret = -1;\n            goto err;\n        }\n        if (!BN_mod(j, key->d, i, ctx)) {\n            ret = -1;\n            goto err;\n        }\n        if (BN_cmp(j, pinfo->d) != 0) {\n            ret = 0;\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_MP_EXPONENT_NOT_CONGRUENT_TO_D);\n        }\n        if (!BN_mod_inverse(i, pinfo->pp, pinfo->r, ctx)) {\n            ret = -1;\n            goto err;\n        }\n        if (BN_cmp(i, pinfo->t) != 0) {\n            ret = 0;\n            RSAerr(RSA_F_RSA_CHECK_KEY_EX, RSA_R_MP_COEFFICIENT_NOT_INVERSE_OF_R);\n        }\n    }\n err:\n    BN_free(i);\n    BN_free(j);\n    BN_free(k);\n    BN_free(l);\n    BN_free(m);\n    BN_CTX_free(ctx);\n    return ret;\n#endif\n}', 'int BN_is_one(const BIGNUM *a)\n{\n    return BN_abs_is_word(a, 1) && !a->neg;\n}', 'int BN_abs_is_word(const BIGNUM *a, const BN_ULONG w)\n{\n    return ((a->top == 1) && (a->d[0] == w)) || ((w == 0) && (a->top == 0));\n}', 'int BN_is_odd(const BIGNUM *a)\n{\n    return (a->top > 0) && (a->d[0] & 1);\n}', 'int BN_mod_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m,\n               BN_CTX *ctx)\n{\n    BIGNUM *t;\n    int ret = 0;\n    bn_check_top(a);\n    bn_check_top(b);\n    bn_check_top(m);\n    BN_CTX_start(ctx);\n    if ((t = BN_CTX_get(ctx)) == NULL)\n        goto err;\n    if (a == b) {\n        if (!BN_sqr(t, a, ctx))\n            goto err;\n    } else {\n        if (!BN_mul(t, a, b, ctx))\n            goto err;\n    }\n    if (!BN_nnmod(r, t, m, ctx))\n        goto err;\n    bn_check_top(r);\n    ret = 1;\n err:\n    BN_CTX_end(ctx);\n    return ret;\n}', 'int BN_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)\n{\n    int ret = bn_mul_fixed_top(r, a, b, ctx);\n    bn_correct_top(r);\n    bn_check_top(r);\n    return ret;\n}', 'int bn_mul_fixed_top(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)\n{\n    int ret = 0;\n    int top, al, bl;\n    BIGNUM *rr;\n#if defined(BN_MUL_COMBA) || defined(BN_RECURSION)\n    int i;\n#endif\n#ifdef BN_RECURSION\n    BIGNUM *t = NULL;\n    int j = 0, k;\n#endif\n    bn_check_top(a);\n    bn_check_top(b);\n    bn_check_top(r);\n    al = a->top;\n    bl = b->top;\n    if ((al == 0) || (bl == 0)) {\n        BN_zero(r);\n        return 1;\n    }\n    top = al + bl;\n    BN_CTX_start(ctx);\n    if ((r == a) || (r == b)) {\n        if ((rr = BN_CTX_get(ctx)) == NULL)\n            goto err;\n    } else\n        rr = r;\n#if defined(BN_MUL_COMBA) || defined(BN_RECURSION)\n    i = al - bl;\n#endif\n#ifdef BN_MUL_COMBA\n    if (i == 0) {\n# if 0\n        if (al == 4) {\n            if (bn_wexpand(rr, 8) == NULL)\n                goto err;\n            rr->top = 8;\n            bn_mul_comba4(rr->d, a->d, b->d);\n            goto end;\n        }\n# endif\n        if (al == 8) {\n            if (bn_wexpand(rr, 16) == NULL)\n                goto err;\n            rr->top = 16;\n            bn_mul_comba8(rr->d, a->d, b->d);\n            goto end;\n        }\n    }\n#endif\n#ifdef BN_RECURSION\n    if ((al >= BN_MULL_SIZE_NORMAL) && (bl >= BN_MULL_SIZE_NORMAL)) {\n        if (i >= -1 && i <= 1) {\n            if (i >= 0) {\n                j = BN_num_bits_word((BN_ULONG)al);\n            }\n            if (i == -1) {\n                j = BN_num_bits_word((BN_ULONG)bl);\n            }\n            j = 1 << (j - 1);\n            assert(j <= al || j <= bl);\n            k = j + j;\n            t = BN_CTX_get(ctx);\n            if (t == NULL)\n                goto err;\n            if (al > j || bl > j) {\n                if (bn_wexpand(t, k * 4) == NULL)\n                    goto err;\n                if (bn_wexpand(rr, k * 4) == NULL)\n                    goto err;\n                bn_mul_part_recursive(rr->d, a->d, b->d,\n                                      j, al - j, bl - j, t->d);\n            } else {\n                if (bn_wexpand(t, k * 2) == NULL)\n                    goto err;\n                if (bn_wexpand(rr, k * 2) == NULL)\n                    goto err;\n                bn_mul_recursive(rr->d, a->d, b->d, j, al - j, bl - j, t->d);\n            }\n            rr->top = top;\n            goto end;\n        }\n    }\n#endif\n    if (bn_wexpand(rr, top) == NULL)\n        goto err;\n    rr->top = top;\n    bn_mul_normal(rr->d, a->d, al, b->d, bl);\n#if defined(BN_MUL_COMBA) || defined(BN_RECURSION)\n end:\n#endif\n    rr->neg = a->neg ^ b->neg;\n    rr->flags |= BN_FLG_FIXED_TOP;\n    if (r != rr && BN_copy(r, rr) == NULL)\n        goto err;\n    ret = 1;\n err:\n    bn_check_top(r);\n    BN_CTX_end(ctx);\n    return ret;\n}', 'int BN_num_bits_word(BN_ULONG l)\n{\n    BN_ULONG x, mask;\n    int bits = (l != 0);\n#if BN_BITS2 > 32\n    x = l >> 32;\n    mask = (0 - x) & BN_MASK2;\n    mask = (0 - (mask >> (BN_BITS2 - 1)));\n    bits += 32 & mask;\n    l ^= (x ^ l) & mask;\n#endif\n    x = l >> 16;\n    mask = (0 - x) & BN_MASK2;\n    mask = (0 - (mask >> (BN_BITS2 - 1)));\n    bits += 16 & mask;\n    l ^= (x ^ l) & mask;\n    x = l >> 8;\n    mask = (0 - x) & BN_MASK2;\n    mask = (0 - (mask >> (BN_BITS2 - 1)));\n    bits += 8 & mask;\n    l ^= (x ^ l) & mask;\n    x = l >> 4;\n    mask = (0 - x) & BN_MASK2;\n    mask = (0 - (mask >> (BN_BITS2 - 1)));\n    bits += 4 & mask;\n    l ^= (x ^ l) & mask;\n    x = l >> 2;\n    mask = (0 - x) & BN_MASK2;\n    mask = (0 - (mask >> (BN_BITS2 - 1)));\n    bits += 2 & mask;\n    l ^= (x ^ l) & mask;\n    x = l >> 1;\n    mask = (0 - x) & BN_MASK2;\n    mask = (0 - (mask >> (BN_BITS2 - 1)));\n    bits += 1 & mask;\n    return bits;\n}']