crypto/rsa/rsa_sp800_56b_gen.c:97: error: BUFFER_OVERRUN_L3
  Offset added: [8, +oo] Size: [0, 536870848] by call to `bn_rsa_fips186_4_gen_prob_primes`.
Showing all 22 steps of the trace


crypto/rsa/rsa_sp800_56b_gen.c:51:1: Parameter `e->top`
49.    *     (Required for CAVS testing).
50.    */
51. > int rsa_fips186_4_gen_prob_primes(RSA *rsa, BIGNUM *p1, BIGNUM *p2,
52.                                     BIGNUM *Xpout, const BIGNUM *Xp,
53.                                     const BIGNUM *Xp1, const BIGNUM *Xp2,

crypto/rsa/rsa_sp800_56b_gen.c:71:10: Call
69.       }
70.   
71.       if (!rsa_check_public_exponent(e)) {
               ^
72.           RSAerr(RSA_F_RSA_FIPS186_4_GEN_PROB_PRIMES,
73.                  RSA_R_PUB_EXPONENT_OUT_OF_RANGE);

crypto/rsa/rsa_sp800_56b_check.c:191:1: Parameter `e->top`
  189.   
  190.   /* Check exponent is odd, and has a bitlen ranging from [17..256] */
  191. > int rsa_check_public_exponent(const BIGNUM *e)
  192.   {
  193.       int bitlen = BN_num_bits(e);

crypto/rsa/rsa_sp800_56b_check.c:193:18: Call
  191.   int rsa_check_public_exponent(const BIGNUM *e)
  192.   {
  193.       int bitlen = BN_num_bits(e);
                          ^
  194.   
  195.       return (BN_is_odd(e) &&  bitlen > 16 && bitlen < 257);

crypto/bn/bn_lib.c:140:9: Call
    138.       bn_check_top(a);
    139.   
    140.       if (BN_is_zero(a))
                   ^
    141.           return 0;
    142.       return ((i * BN_BITS2) + BN_num_bits_word(a->d[i]));

crypto/bn/bn_lib.c:866:1: Parameter `a->top`
      864.   }
      865.   
      866. > int BN_is_zero(const BIGNUM *a)
      867.   {
      868.       return a->top == 0;

crypto/rsa/rsa_sp800_56b_check.c:195:13: Call
  193.       int bitlen = BN_num_bits(e);
  194.   
  195.       return (BN_is_odd(e) &&  bitlen > 16 && bitlen < 257);
                     ^
  196.   }
  197.   

crypto/bn/bn_lib.c:881:1: Parameter `a->top`
    879.   }
    880.   
    881. > int BN_is_odd(const BIGNUM *a)
    882.   {
    883.       return (a->top > 0) && (a->d[0] & 1);

crypto/rsa/rsa_sp800_56b_gen.c:97:10: Call
95.   
96.       /* (Step 4) Generate p, Xp */
97.       if (!bn_rsa_fips186_4_gen_prob_primes(rsa->p, Xpo, p1, p2, Xp, Xp1, Xp2,
                ^
98.                                             nbits, e, ctx, cb))
99.           goto err;

crypto/bn/bn_rsa_fips186_4.c:167:1: Parameter `e->top`
  165.    * Returns: 1 on success otherwise it returns 0.
  166.    */
  167. > int bn_rsa_fips186_4_gen_prob_primes(BIGNUM *p, BIGNUM *Xpout,
  168.                                        BIGNUM *p1, BIGNUM *p2,
  169.                                        const BIGNUM *Xp, const BIGNUM *Xp1,

crypto/bn/bn_rsa_fips186_4.c:215:10: Call
  213.           goto err;
  214.       /* (Steps 4.3/5.3) - generate prime */
  215.       if (!bn_rsa_fips186_4_derive_prime(p, Xpout, Xp, p1i, p2i, nlen, e, ctx, cb))
                  ^
  216.           goto err;
  217.       ret = 1;

crypto/bn/bn_rsa_fips186_4.c:253:1: Parameter `e->top`
    251.    *     Y, X, r1, r2, e are not NULL.
    252.    */
    253. > int bn_rsa_fips186_4_derive_prime(BIGNUM *Y, BIGNUM *X, const BIGNUM *Xin,
    254.                                     const BIGNUM *r1, const BIGNUM *r2, int nlen,
    255.                                     const BIGNUM *e, BN_CTX *ctx, BN_GENCB *cb)

crypto/bn/bn_rsa_fips186_4.c:329:25: Call
    327.               if (BN_copy(y1, Y) == NULL
    328.                       || !BN_sub_word(y1, 1)
    329.                       || !BN_gcd(tmp, y1, e, ctx))
                                   ^
    330.                   goto err;
    331.               if (BN_is_one(tmp)

crypto/bn/bn_gcd.c:15:1: Parameter `in_b->top`
      13.   static BIGNUM *euclid(BIGNUM *a, BIGNUM *b);
      14.   
      15. > int BN_gcd(BIGNUM *r, const BIGNUM *in_a, const BIGNUM *in_b, BN_CTX *ctx)
      16.   {
      17.       BIGNUM *a, *b, *t;

crypto/bn/bn_gcd.c:31:9: Call
      29.       if (BN_copy(a, in_a) == NULL)
      30.           goto err;
      31.       if (BN_copy(b, in_b) == NULL)
                    ^
      32.           goto err;
      33.       a->neg = 0;

crypto/bn/bn_lib.c:281:1: <Offset trace>
279.   }
280.   
281. > BIGNUM *BN_copy(BIGNUM *a, const BIGNUM *b)
282.   {
283.       bn_check_top(b);

crypto/bn/bn_lib.c:281:1: Parameter `b->top`
        279.   }
        280.   
        281. > BIGNUM *BN_copy(BIGNUM *a, const BIGNUM *b)
        282.   {
        283.       bn_check_top(b);

crypto/bn/bn_lib.c:281:1: <Length trace>
279.   }
280.   
281. > BIGNUM *BN_copy(BIGNUM *a, const BIGNUM *b)
282.   {
283.       bn_check_top(b);

crypto/bn/bn_lib.c:281:1: Parameter `*a->d`
        279.   }
        280.   
        281. > BIGNUM *BN_copy(BIGNUM *a, const BIGNUM *b)
        282.   {
        283.       bn_check_top(b);

crypto/bn/bn_lib.c:287:9: Call
        285.       if (a == b)
        286.           return a;
        287.       if (bn_wexpand(a, b->top) == NULL)
                       ^
        288.           return NULL;
        289.   

crypto/bn/bn_lib.c:962:1: Parameter `*a->d`
          960.   }
          961.   
          962. > BIGNUM *bn_wexpand(BIGNUM *a, int words)
          963.   {
          964.       return (words <= a->dmax) ? a : bn_expand2(a, words);

crypto/bn/bn_lib.c:291:9: Array access: Offset added: [8, +oo] Size: [0, 536870848] by call to `bn_rsa_fips186_4_gen_prob_primes` 
        289.   
        290.       if (b->top > 0)
        291.           memcpy(a->d, b->d, sizeof(b->d[0]) * b->top);
                       ^
        292.   
        293.       a->neg = b->neg;

BIGNUM *BN_copy(BIGNUM *a, const BIGNUM *b)
{
    bn_check_top(b);
    if (a == b)
        return a;
    if (bn_wexpand(a, b->top) == NULL)
        return NULL;
    if (b->top > 0)
        memcpy(a->d, b->d, sizeof(b->d[0]) * b->top);
    a->neg = b->neg;
    a->top = b->top;
    a->flags |= b->flags & BN_FLG_FIXED_TOP;
    bn_check_top(a);
    return a;
}
['int rsa_fips186_4_gen_prob_primes(RSA *rsa, BIGNUM *p1, BIGNUM *p2,\n                                  BIGNUM *Xpout, const BIGNUM *Xp,\n                                  const BIGNUM *Xp1, const BIGNUM *Xp2,\n                                  BIGNUM *q1, BIGNUM *q2, BIGNUM *Xqout,\n                                  const BIGNUM *Xq, const BIGNUM *Xq1,\n                                  const BIGNUM *Xq2, int nbits,\n                                  const BIGNUM *e, BN_CTX *ctx, BN_GENCB *cb)\n{\n    int ret = 0, ok;\n    BIGNUM *Xpo = NULL, *Xqo = NULL, *tmp = NULL;\n    if (nbits < RSA_FIPS1864_MIN_KEYGEN_KEYSIZE) {\n        RSAerr(RSA_F_RSA_FIPS186_4_GEN_PROB_PRIMES, RSA_R_INVALID_KEY_LENGTH);\n        return 0;\n    }\n    if (!rsa_check_public_exponent(e)) {\n        RSAerr(RSA_F_RSA_FIPS186_4_GEN_PROB_PRIMES,\n               RSA_R_PUB_EXPONENT_OUT_OF_RANGE);\n        return 0;\n    }\n    BN_CTX_start(ctx);\n    tmp = BN_CTX_get(ctx);\n    Xpo = (Xpout != NULL) ? Xpout : BN_CTX_get(ctx);\n    Xqo = (Xqout != NULL) ? Xqout : BN_CTX_get(ctx);\n    if (tmp == NULL || Xpo == NULL || Xqo == NULL)\n        goto err;\n    if (rsa->p == NULL)\n        rsa->p = BN_secure_new();\n    if (rsa->q == NULL)\n        rsa->q = BN_secure_new();\n    if (rsa->p == NULL || rsa->q == NULL)\n        goto err;\n    if (!bn_rsa_fips186_4_gen_prob_primes(rsa->p, Xpo, p1, p2, Xp, Xp1, Xp2,\n                                          nbits, e, ctx, cb))\n        goto err;\n    for(;;) {\n        if (!bn_rsa_fips186_4_gen_prob_primes(rsa->q, Xqo, q1, q2, Xq, Xq1,\n                                              Xq2, nbits, e, ctx, cb))\n            goto err;\n        ok = rsa_check_pminusq_diff(tmp, Xpo, Xqo, nbits);\n        if (ok < 0)\n            goto err;\n        if (ok == 0)\n            continue;\n        ok = rsa_check_pminusq_diff(tmp, rsa->p, rsa->q, nbits);\n        if (ok < 0)\n            goto err;\n        if (ok == 0)\n            continue;\n        break;\n    }\n    ret = 1;\nerr:\n    if (Xpo != Xpout)\n        BN_clear(Xpo);\n    if (Xqo != Xqout)\n        BN_clear(Xqo);\n    BN_clear(tmp);\n    BN_CTX_end(ctx);\n    return ret;\n}', 'int rsa_check_public_exponent(const BIGNUM *e)\n{\n    int bitlen = BN_num_bits(e);\n    return (BN_is_odd(e) &&  bitlen > 16 && bitlen < 257);\n}', 'int BN_num_bits(const BIGNUM *a)\n{\n    int i = a->top - 1;\n    bn_check_top(a);\n    if (BN_is_zero(a))\n        return 0;\n    return ((i * BN_BITS2) + BN_num_bits_word(a->d[i]));\n}', 'int BN_is_zero(const BIGNUM *a)\n{\n    return a->top == 0;\n}', 'int BN_is_odd(const BIGNUM *a)\n{\n    return (a->top > 0) && (a->d[0] & 1);\n}', 'int bn_rsa_fips186_4_gen_prob_primes(BIGNUM *p, BIGNUM *Xpout,\n                                     BIGNUM *p1, BIGNUM *p2,\n                                     const BIGNUM *Xp, const BIGNUM *Xp1,\n                                     const BIGNUM *Xp2, int nlen,\n                                     const BIGNUM *e, BN_CTX *ctx, BN_GENCB *cb)\n{\n    int ret = 0;\n    BIGNUM *p1i = NULL, *p2i = NULL, *Xp1i = NULL, *Xp2i = NULL;\n    int bitlen;\n    if (p == NULL || Xpout == NULL)\n        return 0;\n    BN_CTX_start(ctx);\n    p1i = (p1 != NULL) ? p1 : BN_CTX_get(ctx);\n    p2i = (p2 != NULL) ? p2 : BN_CTX_get(ctx);\n    Xp1i = (Xp1 != NULL) ? (BIGNUM *)Xp1 : BN_CTX_get(ctx);\n    Xp2i = (Xp2 != NULL) ? (BIGNUM *)Xp2 : BN_CTX_get(ctx);\n    if (p1i == NULL || p2i == NULL || Xp1i == NULL || Xp2i == NULL)\n        goto err;\n    bitlen = bn_rsa_fips186_4_aux_prime_min_size(nlen);\n    if (bitlen == 0)\n        goto err;\n    if (Xp1 == NULL) {\n        if (!BN_priv_rand(Xp1i, bitlen, BN_RAND_TOP_ONE, BN_RAND_BOTTOM_ODD))\n            goto err;\n    }\n    if (Xp2 == NULL) {\n        if (!BN_priv_rand(Xp2i, bitlen, BN_RAND_TOP_ONE, BN_RAND_BOTTOM_ODD))\n            goto err;\n    }\n    if (!bn_rsa_fips186_4_find_aux_prob_prime(Xp1i, p1i, ctx, cb)\n            || !bn_rsa_fips186_4_find_aux_prob_prime(Xp2i, p2i, ctx, cb))\n        goto err;\n    if ((BN_num_bits(p1i) + BN_num_bits(p2i)) >=\n            bn_rsa_fips186_4_aux_prime_max_sum_size_for_prob_primes(nlen))\n        goto err;\n    if (!bn_rsa_fips186_4_derive_prime(p, Xpout, Xp, p1i, p2i, nlen, e, ctx, cb))\n        goto err;\n    ret = 1;\nerr:\n    if (p1 == NULL)\n        BN_clear(p1i);\n    if (p2 == NULL)\n        BN_clear(p2i);\n    if (Xp1 == NULL)\n        BN_clear(Xp1i);\n    if (Xp2 == NULL)\n        BN_clear(Xp2i);\n    BN_CTX_end(ctx);\n    return ret;\n}', 'int bn_rsa_fips186_4_derive_prime(BIGNUM *Y, BIGNUM *X, const BIGNUM *Xin,\n                                  const BIGNUM *r1, const BIGNUM *r2, int nlen,\n                                  const BIGNUM *e, BN_CTX *ctx, BN_GENCB *cb)\n{\n    int ret = 0;\n    int i, imax;\n    int bits = nlen >> 1;\n    int checks = bn_rsa_fips186_4_prime_MR_min_checks(nlen);\n    BIGNUM *tmp, *R, *r1r2x2, *y1, *r1x2;\n    if (checks == 0)\n        return 0;\n    BN_CTX_start(ctx);\n    R = BN_CTX_get(ctx);\n    tmp = BN_CTX_get(ctx);\n    r1r2x2 = BN_CTX_get(ctx);\n    y1 = BN_CTX_get(ctx);\n    r1x2 = BN_CTX_get(ctx);\n    if (r1x2 == NULL)\n        goto err;\n    if (Xin != NULL && BN_copy(X, Xin) == NULL)\n        goto err;\n    if (!(BN_lshift1(r1x2, r1)\n            && BN_gcd(tmp, r1x2, r2, ctx)\n            && BN_is_one(tmp)\n            && BN_mod_inverse(R, r2, r1x2, ctx)\n            && BN_mul(R, R, r2, ctx)\n            && BN_mod_inverse(tmp, r1x2, r2, ctx)\n            && BN_mul(tmp, tmp, r1x2, ctx)\n            && BN_sub(R, R, tmp)\n            && BN_mul(r1r2x2, r1x2, r2, ctx)))\n        goto err;\n    if (BN_is_negative(R) && !BN_add(R, R, r1r2x2))\n        goto err;\n    imax = 5 * bits;\n    for (;;) {\n        if (Xin == NULL) {\n            if (!BN_priv_rand(X, bits, BN_RAND_TOP_TWO, BN_RAND_BOTTOM_ANY))\n                goto end;\n        }\n        if (!BN_mod_sub(Y, R, X, r1r2x2, ctx) || !BN_add(Y, Y, X))\n            goto err;\n        i = 0;\n        for (;;) {\n            if (BN_num_bits(Y) > bits) {\n                if (Xin == NULL)\n                    break;\n                else\n                    goto err;\n            }\n            BN_GENCB_call(cb, 0, 2);\n            if (BN_copy(y1, Y) == NULL\n                    || !BN_sub_word(y1, 1)\n                    || !BN_gcd(tmp, y1, e, ctx))\n                goto err;\n            if (BN_is_one(tmp)\n                    && BN_is_prime_fasttest_ex(Y, checks, ctx, 1, cb))\n                goto end;\n            if (++i >= imax || !BN_add(Y, Y, r1r2x2))\n                goto err;\n        }\n    }\nend:\n    ret = 1;\n    BN_GENCB_call(cb, 3, 0);\nerr:\n    BN_clear(y1);\n    BN_CTX_end(ctx);\n    return ret;\n}', 'int BN_gcd(BIGNUM *r, const BIGNUM *in_a, const BIGNUM *in_b, BN_CTX *ctx)\n{\n    BIGNUM *a, *b, *t;\n    int ret = 0;\n    bn_check_top(in_a);\n    bn_check_top(in_b);\n    BN_CTX_start(ctx);\n    a = BN_CTX_get(ctx);\n    b = BN_CTX_get(ctx);\n    if (b == NULL)\n        goto err;\n    if (BN_copy(a, in_a) == NULL)\n        goto err;\n    if (BN_copy(b, in_b) == NULL)\n        goto err;\n    a->neg = 0;\n    b->neg = 0;\n    if (BN_cmp(a, b) < 0) {\n        t = a;\n        a = b;\n        b = t;\n    }\n    t = euclid(a, b);\n    if (t == NULL)\n        goto err;\n    if (BN_copy(r, t) == NULL)\n        goto err;\n    ret = 1;\n err:\n    BN_CTX_end(ctx);\n    bn_check_top(r);\n    return ret;\n}', 'BIGNUM *BN_copy(BIGNUM *a, const BIGNUM *b)\n{\n    bn_check_top(b);\n    if (a == b)\n        return a;\n    if (bn_wexpand(a, b->top) == NULL)\n        return NULL;\n    if (b->top > 0)\n        memcpy(a->d, b->d, sizeof(b->d[0]) * b->top);\n    a->neg = b->neg;\n    a->top = b->top;\n    a->flags |= b->flags & BN_FLG_FIXED_TOP;\n    bn_check_top(a);\n    return a;\n}', 'BIGNUM *bn_wexpand(BIGNUM *a, int words)\n{\n    return (words <= a->dmax) ? a : bn_expand2(a, words);\n}']