crypto/rsa/rsa_sp800_56b_gen.c:97: error: BUFFER_OVERRUN_L3
  Offset added: [8, +oo] Size: [0, 536870848] by call to `bn_rsa_fips186_4_gen_prob_primes`.
Showing all 17 steps of the trace


crypto/rsa/rsa_sp800_56b_gen.c:51:1: Parameter `Xpout->d`
49.    *     (Required for CAVS testing).
50.    */
51. > int rsa_fips186_4_gen_prob_primes(RSA *rsa, BIGNUM *p1, BIGNUM *p2,
52.                                     BIGNUM *Xpout, const BIGNUM *Xp,
53.                                     const BIGNUM *Xp1, const BIGNUM *Xp2,

crypto/rsa/rsa_sp800_56b_gen.c:97:10: Call
95.   
96.       /* (Step 4) Generate p, Xp */
97.       if (!bn_rsa_fips186_4_gen_prob_primes(rsa->p, Xpo, p1, p2, Xp, Xp1, Xp2,
                ^
98.                                             nbits, e, ctx, cb))
99.           goto err;

crypto/bn/bn_rsa_fips186_4.c:167:1: Parameter `*Xpout->d`
  165.    * Returns: 1 on success otherwise it returns 0.
  166.    */
  167. > int bn_rsa_fips186_4_gen_prob_primes(BIGNUM *p, BIGNUM *Xpout,
  168.                                        BIGNUM *p1, BIGNUM *p2,
  169.                                        const BIGNUM *Xp, const BIGNUM *Xp1,

crypto/bn/bn_rsa_fips186_4.c:215:10: Call
  213.           goto err;
  214.       /* (Steps 4.3/5.3) - generate prime */
  215.       if (!bn_rsa_fips186_4_derive_prime(p, Xpout, Xp, p1i, p2i, nlen, e, ctx, cb))
                  ^
  216.           goto err;
  217.       ret = 1;

crypto/bn/bn_rsa_fips186_4.c:253:1: Parameter `*X->d`
    251.    *     Y, X, r1, r2, e are not NULL.
    252.    */
    253. > int bn_rsa_fips186_4_derive_prime(BIGNUM *Y, BIGNUM *X, const BIGNUM *Xin,
    254.                                     const BIGNUM *r1, const BIGNUM *r2, int nlen,
    255.                                     const BIGNUM *e, BN_CTX *ctx, BN_GENCB *cb)

crypto/bn/bn_rsa_fips186_4.c:275:24: Call
    273.           goto err;
    274.   
    275.       if (Xin != NULL && BN_copy(X, Xin) == NULL)
                                  ^
    276.           goto err;
    277.   

crypto/bn/bn_lib.c:281:1: Parameter `*a->d`
      279.   }
      280.   
      281. > BIGNUM *BN_copy(BIGNUM *a, const BIGNUM *b)
      282.   {
      283.       bn_check_top(b);

crypto/bn/bn_lib.c:287:9: Call
      285.       if (a == b)
      286.           return a;
      287.       if (bn_wexpand(a, b->top) == NULL)
                     ^
      288.           return NULL;
      289.   

crypto/bn/bn_lib.c:960:1: Parameter `*a->d`
        958.   }
        959.   
        960. > BIGNUM *bn_wexpand(BIGNUM *a, int words)
        961.   {
        962.       return (words <= a->dmax) ? a : bn_expand2(a, words);

crypto/bn/bn_lib.c:962:37: Call
        960.   BIGNUM *bn_wexpand(BIGNUM *a, int words)
        961.   {
        962.       return (words <= a->dmax) ? a : bn_expand2(a, words);
                                                   ^
        963.   }
        964.   

crypto/bn/bn_lib.c:245:1: Parameter `*b->d`
          243.    */
          244.   
          245. > BIGNUM *bn_expand2(BIGNUM *b, int words)
          246.   {
          247.       if (words > b->dmax) {

crypto/bn/bn_lib.c:248:23: Call
          246.   {
          247.       if (words > b->dmax) {
          248.           BN_ULONG *a = bn_expand_internal(b, words);
                                       ^
          249.           if (!a)
          250.               return NULL;

crypto/bn/bn_lib.c:209:1: <Offset trace>
207.   /* This is used by bn_expand2() */
208.   /* The caller MUST check that words > b->dmax before calling this */
209. > static BN_ULONG *bn_expand_internal(const BIGNUM *b, int words)
210.   {
211.       BN_ULONG *a = NULL;

crypto/bn/bn_lib.c:209:1: Parameter `b->top`
            207.   /* This is used by bn_expand2() */
            208.   /* The caller MUST check that words > b->dmax before calling this */
            209. > static BN_ULONG *bn_expand_internal(const BIGNUM *b, int words)
            210.   {
            211.       BN_ULONG *a = NULL;

crypto/bn/bn_lib.c:209:1: <Length trace>
207.   /* This is used by bn_expand2() */
208.   /* The caller MUST check that words > b->dmax before calling this */
209. > static BN_ULONG *bn_expand_internal(const BIGNUM *b, int words)
210.   {
211.       BN_ULONG *a = NULL;

crypto/bn/bn_lib.c:209:1: Parameter `*b->d`
            207.   /* This is used by bn_expand2() */
            208.   /* The caller MUST check that words > b->dmax before calling this */
            209. > static BN_ULONG *bn_expand_internal(const BIGNUM *b, int words)
            210.   {
            211.       BN_ULONG *a = NULL;

crypto/bn/bn_lib.c:232:9: Array access: Offset added: [8, +oo] Size: [0, 536870848] by call to `bn_rsa_fips186_4_gen_prob_primes` 
            230.       assert(b->top <= words);
            231.       if (b->top > 0)
            232.           memcpy(a, b->d, sizeof(*a) * b->top);
                           ^
            233.   
            234.       return a;

static BN_ULONG *bn_expand_internal(const BIGNUM *b, int words)
{
    BN_ULONG *a = NULL;
    if (words > (INT_MAX / (4 * BN_BITS2))) {
        BNerr(BN_F_BN_EXPAND_INTERNAL, BN_R_BIGNUM_TOO_LONG);
        return NULL;
    }
    if (BN_get_flags(b, BN_FLG_STATIC_DATA)) {
        BNerr(BN_F_BN_EXPAND_INTERNAL, BN_R_EXPAND_ON_STATIC_BIGNUM_DATA);
        return NULL;
    }
    if (BN_get_flags(b, BN_FLG_SECURE))
        a = OPENSSL_secure_zalloc(words * sizeof(*a));
    else
        a = OPENSSL_zalloc(words * sizeof(*a));
    if (a == NULL) {
        BNerr(BN_F_BN_EXPAND_INTERNAL, ERR_R_MALLOC_FAILURE);
        return NULL;
    }
    assert(b->top <= words);
    if (b->top > 0)
        memcpy(a, b->d, sizeof(*a) * b->top);
    return a;
}
['int rsa_fips186_4_gen_prob_primes(RSA *rsa, BIGNUM *p1, BIGNUM *p2,\n                                  BIGNUM *Xpout, const BIGNUM *Xp,\n                                  const BIGNUM *Xp1, const BIGNUM *Xp2,\n                                  BIGNUM *q1, BIGNUM *q2, BIGNUM *Xqout,\n                                  const BIGNUM *Xq, const BIGNUM *Xq1,\n                                  const BIGNUM *Xq2, int nbits,\n                                  const BIGNUM *e, BN_CTX *ctx, BN_GENCB *cb)\n{\n    int ret = 0, ok;\n    BIGNUM *Xpo = NULL, *Xqo = NULL, *tmp = NULL;\n    if (nbits < RSA_FIPS1864_MIN_KEYGEN_KEYSIZE) {\n        RSAerr(RSA_F_RSA_FIPS186_4_GEN_PROB_PRIMES, RSA_R_INVALID_KEY_LENGTH);\n        return 0;\n    }\n    if (!rsa_check_public_exponent(e)) {\n        RSAerr(RSA_F_RSA_FIPS186_4_GEN_PROB_PRIMES,\n               RSA_R_PUB_EXPONENT_OUT_OF_RANGE);\n        goto err;\n    }\n    BN_CTX_start(ctx);\n    tmp = BN_CTX_get(ctx);\n    Xpo = (Xpout != NULL) ? Xpout : BN_CTX_get(ctx);\n    Xqo = (Xqout != NULL) ? Xqout : BN_CTX_get(ctx);\n    if (tmp == NULL || Xpo == NULL || Xqo == NULL)\n        goto err;\n    if (rsa->p == NULL)\n        rsa->p = BN_secure_new();\n    if (rsa->q == NULL)\n        rsa->q = BN_secure_new();\n    if (rsa->p == NULL || rsa->q == NULL)\n        goto err;\n    if (!bn_rsa_fips186_4_gen_prob_primes(rsa->p, Xpo, p1, p2, Xp, Xp1, Xp2,\n                                          nbits, e, ctx, cb))\n        goto err;\n    for(;;) {\n        if (!bn_rsa_fips186_4_gen_prob_primes(rsa->q, Xqo, q1, q2, Xq, Xq1,\n                                              Xq2, nbits, e, ctx, cb))\n            goto err;\n        ok = rsa_check_pminusq_diff(tmp, Xpo, Xqo, nbits);\n        if (ok < 0)\n            goto err;\n        if (ok == 0)\n            continue;\n        ok = rsa_check_pminusq_diff(tmp, rsa->p, rsa->q, nbits);\n        if (ok < 0)\n            goto err;\n        if (ok == 0)\n            continue;\n        break;\n    }\n    ret = 1;\nerr:\n    if (Xpo != Xpout)\n        BN_clear(Xpo);\n    if (Xqo != Xqout)\n        BN_clear(Xqo);\n    BN_clear(tmp);\n    BN_CTX_end(ctx);\n    return ret;\n}', 'int bn_rsa_fips186_4_gen_prob_primes(BIGNUM *p, BIGNUM *Xpout,\n                                     BIGNUM *p1, BIGNUM *p2,\n                                     const BIGNUM *Xp, const BIGNUM *Xp1,\n                                     const BIGNUM *Xp2, int nlen,\n                                     const BIGNUM *e, BN_CTX *ctx, BN_GENCB *cb)\n{\n    int ret = 0;\n    BIGNUM *p1i = NULL, *p2i = NULL, *Xp1i = NULL, *Xp2i = NULL;\n    int bitlen;\n    if (p == NULL || Xpout == NULL)\n        return 0;\n    BN_CTX_start(ctx);\n    p1i = (p1 != NULL) ? p1 : BN_CTX_get(ctx);\n    p2i = (p2 != NULL) ? p2 : BN_CTX_get(ctx);\n    Xp1i = (Xp1 != NULL) ? (BIGNUM *)Xp1 : BN_CTX_get(ctx);\n    Xp2i = (Xp2 != NULL) ? (BIGNUM *)Xp2 : BN_CTX_get(ctx);\n    if (p1i == NULL || p2i == NULL || Xp1i == NULL || Xp2i == NULL)\n        goto err;\n    bitlen = bn_rsa_fips186_4_aux_prime_min_size(nlen);\n    if (bitlen == 0)\n        goto err;\n    if (Xp1 == NULL) {\n        if (!BN_priv_rand(Xp1i, bitlen, BN_RAND_TOP_ONE, BN_RAND_BOTTOM_ODD))\n            goto err;\n    }\n    if (Xp2 == NULL) {\n        if (!BN_priv_rand(Xp2i, bitlen, BN_RAND_TOP_ONE, BN_RAND_BOTTOM_ODD))\n            goto err;\n    }\n    if (!bn_rsa_fips186_4_find_aux_prob_prime(Xp1i, p1i, ctx, cb)\n            || !bn_rsa_fips186_4_find_aux_prob_prime(Xp2i, p2i, ctx, cb))\n        goto err;\n    if ((BN_num_bits(p1i) + BN_num_bits(p2i)) >=\n            bn_rsa_fips186_4_aux_prime_max_sum_size_for_prob_primes(nlen))\n        goto err;\n    if (!bn_rsa_fips186_4_derive_prime(p, Xpout, Xp, p1i, p2i, nlen, e, ctx, cb))\n        goto err;\n    ret = 1;\nerr:\n    if (p1 == NULL)\n        BN_clear(p1i);\n    if (p2 == NULL)\n        BN_clear(p2i);\n    if (Xp1 == NULL)\n        BN_clear(Xp1i);\n    if (Xp2 == NULL)\n        BN_clear(Xp2i);\n    BN_CTX_end(ctx);\n    return ret;\n}', 'int bn_rsa_fips186_4_derive_prime(BIGNUM *Y, BIGNUM *X, const BIGNUM *Xin,\n                                  const BIGNUM *r1, const BIGNUM *r2, int nlen,\n                                  const BIGNUM *e, BN_CTX *ctx, BN_GENCB *cb)\n{\n    int ret = 0;\n    int i, imax;\n    int bits = nlen >> 1;\n    int checks = bn_rsa_fips186_4_prime_MR_min_checks(nlen);\n    BIGNUM *tmp, *R, *r1r2x2, *y1, *r1x2;\n    if (checks == 0)\n        return 0;\n    BN_CTX_start(ctx);\n    R = BN_CTX_get(ctx);\n    tmp = BN_CTX_get(ctx);\n    r1r2x2 = BN_CTX_get(ctx);\n    y1 = BN_CTX_get(ctx);\n    r1x2 = BN_CTX_get(ctx);\n    if (r1x2 == NULL)\n        goto err;\n    if (Xin != NULL && BN_copy(X, Xin) == NULL)\n        goto err;\n    if (!(BN_lshift1(r1x2, r1)\n            && BN_gcd(tmp, r1x2, r2, ctx)\n            && BN_is_one(tmp)\n            && BN_mod_inverse(R, r2, r1x2, ctx)\n            && BN_mul(R, R, r2, ctx)\n            && BN_mod_inverse(tmp, r1x2, r2, ctx)\n            && BN_mul(tmp, tmp, r1x2, ctx)\n            && BN_sub(R, R, tmp)\n            && BN_mul(r1r2x2, r1x2, r2, ctx)))\n        goto err;\n    if (BN_is_negative(R) && !BN_add(R, R, r1r2x2))\n        goto err;\n    imax = 5 * bits;\n    for (;;) {\n        if (Xin == NULL) {\n            if (!BN_priv_rand(X, bits, BN_RAND_TOP_TWO, BN_RAND_BOTTOM_ANY))\n                goto end;\n        }\n        if (!BN_mod_sub(Y, R, X, r1r2x2, ctx) || !BN_add(Y, Y, X))\n            goto err;\n        i = 0;\n        for (;;) {\n            if (BN_num_bits(Y) > bits) {\n                if (Xin == NULL)\n                    break;\n                else\n                    goto err;\n            }\n            BN_GENCB_call(cb, 0, 2);\n            if (BN_copy(y1, Y) == NULL\n                    || !BN_sub_word(y1, 1)\n                    || !BN_gcd(tmp, y1, e, ctx))\n                goto err;\n            if (BN_is_one(tmp)\n                    && BN_is_prime_fasttest_ex(Y, checks, ctx, 1, cb))\n                goto end;\n            if (++i >= imax || !BN_add(Y, Y, r1r2x2))\n                goto err;\n        }\n    }\nend:\n    ret = 1;\n    BN_GENCB_call(cb, 3, 0);\nerr:\n    BN_clear(y1);\n    BN_CTX_end(ctx);\n    return ret;\n}', 'BIGNUM *BN_copy(BIGNUM *a, const BIGNUM *b)\n{\n    bn_check_top(b);\n    if (a == b)\n        return a;\n    if (bn_wexpand(a, b->top) == NULL)\n        return NULL;\n    if (b->top > 0)\n        memcpy(a->d, b->d, sizeof(b->d[0]) * b->top);\n    a->neg = b->neg;\n    a->top = b->top;\n    a->flags |= b->flags & BN_FLG_FIXED_TOP;\n    bn_check_top(a);\n    return a;\n}', 'BIGNUM *bn_wexpand(BIGNUM *a, int words)\n{\n    return (words <= a->dmax) ? a : bn_expand2(a, words);\n}', 'BIGNUM *bn_expand2(BIGNUM *b, int words)\n{\n    if (words > b->dmax) {\n        BN_ULONG *a = bn_expand_internal(b, words);\n        if (!a)\n            return NULL;\n        if (b->d) {\n            OPENSSL_cleanse(b->d, b->dmax * sizeof(b->d[0]));\n            bn_free_d(b);\n        }\n        b->d = a;\n        b->dmax = words;\n    }\n    return b;\n}', 'static BN_ULONG *bn_expand_internal(const BIGNUM *b, int words)\n{\n    BN_ULONG *a = NULL;\n    if (words > (INT_MAX / (4 * BN_BITS2))) {\n        BNerr(BN_F_BN_EXPAND_INTERNAL, BN_R_BIGNUM_TOO_LONG);\n        return NULL;\n    }\n    if (BN_get_flags(b, BN_FLG_STATIC_DATA)) {\n        BNerr(BN_F_BN_EXPAND_INTERNAL, BN_R_EXPAND_ON_STATIC_BIGNUM_DATA);\n        return NULL;\n    }\n    if (BN_get_flags(b, BN_FLG_SECURE))\n        a = OPENSSL_secure_zalloc(words * sizeof(*a));\n    else\n        a = OPENSSL_zalloc(words * sizeof(*a));\n    if (a == NULL) {\n        BNerr(BN_F_BN_EXPAND_INTERNAL, ERR_R_MALLOC_FAILURE);\n        return NULL;\n    }\n    assert(b->top <= words);\n    if (b->top > 0)\n        memcpy(a, b->d, sizeof(*a) * b->top);\n    return a;\n}']