crypto/rsa/rsa_gen.c:170: error: BUFFER_OVERRUN_L3
  Offset: [-1, +oo] Size: [1, +oo] by call to `BN_generate_prime_ex`.
Showing all 20 steps of the trace


crypto/rsa/rsa_gen.c:65:1: Parameter `bits`
63.   }
64.   
65. > static int rsa_builtin_keygen(RSA *rsa, int bits, int primes, BIGNUM *e_value,
66.                                 BN_GENCB *cb)
67.   {

crypto/rsa/rsa_gen.c:105:5: Assignment
103.   
104.       /* divide bits into 'primes' pieces evenly */
105.       quo = bits / primes;
           ^
106.       rmd = bits % primes;
107.   

crypto/rsa/rsa_gen.c:109:20: Assignment
107.   
108.       for (i = 0; i < primes; i++)
109.           bitsr[i] = (i < rmd) ? quo + 1 : quo;
                          ^
110.   
111.       /* We need the RSA components non-NULL */

crypto/rsa/rsa_gen.c:109:9: Assignment
107.   
108.       for (i = 0; i < primes; i++)
109.           bitsr[i] = (i < rmd) ? quo + 1 : quo;
               ^
110.   
111.       /* We need the RSA components non-NULL */

crypto/rsa/rsa_gen.c:170:18: Call
168.           for (;;) {
169.    redo:
170.               if (!BN_generate_prime_ex(prime, bitsr[i] + adj, 0, NULL, NULL, cb))
                        ^
171.                   goto err;
172.               /*

crypto/bn/bn_prime.c:87:1: Parameter `bits`
  85.   }
  86.   
  87. > int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe,
  88.                            const BIGNUM *add, const BIGNUM *rem, BN_GENCB *cb)
  89.   {

crypto/bn/bn_prime.c:121:14: Call
  119.       /* make a random number and set the top and bottom bits */
  120.       if (add == NULL) {
  121.           if (!probable_prime(ret, bits, mods))
                      ^
  122.               goto err;
  123.       } else {

crypto/bn/bn_prime.c:379:1: Parameter `bits`
    377.   }
    378.   
    379. > static int probable_prime(BIGNUM *rnd, int bits, prime_t *mods)
    380.   {
    381.       int i;

crypto/bn/bn_prime.c:388:10: Call
    386.    again:
    387.       /* TODO: Not all primes are private */
    388.       if (!BN_priv_rand(rnd, bits, BN_RAND_TOP_TWO, BN_RAND_BOTTOM_ODD))
                    ^
    389.           return 0;
    390.       /* we now have a random number 'rnd' to test. */

crypto/bn/bn_rand.c:107:1: Parameter `bits`
      105.   }
      106.   
      107. > int BN_priv_rand(BIGNUM *rnd, int bits, int top, int bottom)
      108.   {
      109.       return bnrand(PRIVATE, rnd, bits, top, bottom);

crypto/bn/bn_rand.c:109:12: Call
      107.   int BN_priv_rand(BIGNUM *rnd, int bits, int top, int bottom)
      108.   {
      109.       return bnrand(PRIVATE, rnd, bits, top, bottom);
                        ^
      110.   }
      111.   

crypto/bn/bn_rand.c:57:14: <Offset trace>
55.           unsigned char c;
56.   
57.           for (i = 0; i < bytes; i++) {
                   ^
58.               if (RAND_bytes(&c, 1) <= 0)
59.                   goto err;

crypto/bn/bn_rand.c:57:14: Assignment
        55.           unsigned char c;
        56.   
        57.           for (i = 0; i < bytes; i++) {
                           ^
        58.               if (RAND_bytes(&c, 1) <= 0)
        59.                   goto err;

crypto/bn/bn_rand.c:21:1: <Length trace>
19.   } BNRAND_FLAG;
20.   
21. > static int bnrand(BNRAND_FLAG flag, BIGNUM *rnd, int bits, int top, int bottom)
22.   {
23.       unsigned char *buf = NULL;

crypto/bn/bn_rand.c:21:1: Parameter `bits`
        19.   } BNRAND_FLAG;
        20.   
        21. > static int bnrand(BNRAND_FLAG flag, BIGNUM *rnd, int bits, int top, int bottom)
        22.   {
        23.       unsigned char *buf = NULL;

crypto/bn/bn_rand.c:35:5: Assignment
        33.           goto toosmall;
        34.   
        35.       bytes = (bits + 7) / 8;
                  ^
        36.       bit = (bits - 1) % 8;
        37.       mask = 0xff << (bit + 1);

crypto/bn/bn_rand.c:39:11: Call
        37.       mask = 0xff << (bit + 1);
        38.   
        39.       buf = OPENSSL_malloc(bytes);
                        ^
        40.       if (buf == NULL) {
        41.           BNerr(BN_F_BNRAND, ERR_R_MALLOC_FAILURE);

crypto/mem.c:201:9: Assignment
          199.   
          200.       if (num == 0)
          201.           return NULL;
                         ^
          202.   
          203.       FAILTEST();

crypto/bn/bn_rand.c:39:5: Assignment
        37.       mask = 0xff << (bit + 1);
        38.   
        39.       buf = OPENSSL_malloc(bytes);
                  ^
        40.       if (buf == NULL) {
        41.           BNerr(BN_F_BNRAND, ERR_R_MALLOC_FAILURE);

crypto/bn/bn_rand.c:83:9: Array access: Offset: [-1, +oo] Size: [1, +oo] by call to `BN_generate_prime_ex` 
        81.       buf[0] &= ~mask;
        82.       if (bottom)                 /* set bottom bit if requested */
        83.           buf[bytes - 1] |= 1;
                      ^
        84.       if (!BN_bin2bn(buf, bytes, rnd))
        85.           goto err;

static int bnrand(BNRAND_FLAG flag, BIGNUM *rnd, int bits, int top, int bottom)
{
    unsigned char *buf = NULL;
    int b, ret = 0, bit, bytes, mask;
    if (bits == 0) {
        if (top != BN_RAND_TOP_ANY || bottom != BN_RAND_BOTTOM_ANY)
            goto toosmall;
        BN_zero(rnd);
        return 1;
    }
    if (bits < 0 || (bits == 1 && top > 0))
        goto toosmall;
    bytes = (bits + 7) / 8;
    bit = (bits - 1) % 8;
    mask = 0xff << (bit + 1);
    buf = OPENSSL_malloc(bytes);
    if (buf == NULL) {
        BNerr(BN_F_BNRAND, ERR_R_MALLOC_FAILURE);
        goto err;
    }
    b = flag == NORMAL ? RAND_bytes(buf, bytes) : RAND_priv_bytes(buf, bytes);
    if (b <= 0)
        goto err;
    if (flag == TESTING) {
        int i;
        unsigned char c;
        for (i = 0; i < bytes; i++) {
            if (RAND_bytes(&c, 1) <= 0)
                goto err;
            if (c >= 128 && i > 0)
                buf[i] = buf[i - 1];
            else if (c < 42)
                buf[i] = 0;
            else if (c < 84)
                buf[i] = 255;
        }
    }
    if (top >= 0) {
        if (top) {
            if (bit == 0) {
                buf[0] = 1;
                buf[1] |= 0x80;
            } else {
                buf[0] |= (3 << (bit - 1));
            }
        } else {
            buf[0] |= (1 << bit);
        }
    }
    buf[0] &= ~mask;
    if (bottom)
        buf[bytes - 1] |= 1;
    if (!BN_bin2bn(buf, bytes, rnd))
        goto err;
    ret = 1;
 err:
    OPENSSL_clear_free(buf, bytes);
    bn_check_top(rnd);
    return ret;
toosmall:
    BNerr(BN_F_BNRAND, BN_R_BITS_TOO_SMALL);
    return 0;
}
['static int rsa_builtin_keygen(RSA *rsa, int bits, int primes, BIGNUM *e_value,\n                              BN_GENCB *cb)\n{\n#ifdef FIPS_MODE\n    if (primes != 2)\n        return 0;\n    return rsa_sp800_56b_generate_key(rsa, bits, e_value, cb);\n#else\n    BIGNUM *r0 = NULL, *r1 = NULL, *r2 = NULL, *tmp, *prime;\n    int ok = -1, n = 0, bitsr[RSA_MAX_PRIME_NUM], bitse = 0;\n    int i = 0, quo = 0, rmd = 0, adj = 0, retries = 0;\n    RSA_PRIME_INFO *pinfo = NULL;\n    STACK_OF(RSA_PRIME_INFO) *prime_infos = NULL;\n    BN_CTX *ctx = NULL;\n    BN_ULONG bitst = 0;\n    unsigned long error = 0;\n    if (bits < RSA_MIN_MODULUS_BITS) {\n        ok = 0;\n        RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, RSA_R_KEY_SIZE_TOO_SMALL);\n        goto err;\n    }\n    if (primes < RSA_DEFAULT_PRIME_NUM || primes > rsa_multip_cap(bits)) {\n        ok = 0;\n        RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, RSA_R_KEY_PRIME_NUM_INVALID);\n        goto err;\n    }\n    ctx = BN_CTX_new();\n    if (ctx == NULL)\n        goto err;\n    BN_CTX_start(ctx);\n    r0 = BN_CTX_get(ctx);\n    r1 = BN_CTX_get(ctx);\n    r2 = BN_CTX_get(ctx);\n    if (r2 == NULL)\n        goto err;\n    quo = bits / primes;\n    rmd = bits % primes;\n    for (i = 0; i < primes; i++)\n        bitsr[i] = (i < rmd) ? quo + 1 : quo;\n    if (!rsa->n && ((rsa->n = BN_new()) == NULL))\n        goto err;\n    if (!rsa->d && ((rsa->d = BN_secure_new()) == NULL))\n        goto err;\n    if (!rsa->e && ((rsa->e = BN_new()) == NULL))\n        goto err;\n    if (!rsa->p && ((rsa->p = BN_secure_new()) == NULL))\n        goto err;\n    if (!rsa->q && ((rsa->q = BN_secure_new()) == NULL))\n        goto err;\n    if (!rsa->dmp1 && ((rsa->dmp1 = BN_secure_new()) == NULL))\n        goto err;\n    if (!rsa->dmq1 && ((rsa->dmq1 = BN_secure_new()) == NULL))\n        goto err;\n    if (!rsa->iqmp && ((rsa->iqmp = BN_secure_new()) == NULL))\n        goto err;\n    if (primes > RSA_DEFAULT_PRIME_NUM) {\n        rsa->version = RSA_ASN1_VERSION_MULTI;\n        prime_infos = sk_RSA_PRIME_INFO_new_reserve(NULL, primes - 2);\n        if (prime_infos == NULL)\n            goto err;\n        if (rsa->prime_infos != NULL) {\n            sk_RSA_PRIME_INFO_pop_free(rsa->prime_infos, rsa_multip_info_free);\n        }\n        rsa->prime_infos = prime_infos;\n        for (i = 2; i < primes; i++) {\n            pinfo = rsa_multip_info_new();\n            if (pinfo == NULL)\n                goto err;\n            (void)sk_RSA_PRIME_INFO_push(prime_infos, pinfo);\n        }\n    }\n    if (BN_copy(rsa->e, e_value) == NULL)\n        goto err;\n    for (i = 0; i < primes; i++) {\n        adj = 0;\n        retries = 0;\n        if (i == 0) {\n            prime = rsa->p;\n        } else if (i == 1) {\n            prime = rsa->q;\n        } else {\n            pinfo = sk_RSA_PRIME_INFO_value(prime_infos, i - 2);\n            prime = pinfo->r;\n        }\n        BN_set_flags(prime, BN_FLG_CONSTTIME);\n        for (;;) {\n redo:\n            if (!BN_generate_prime_ex(prime, bitsr[i] + adj, 0, NULL, NULL, cb))\n                goto err;\n            {\n                int j;\n                for (j = 0; j < i; j++) {\n                    BIGNUM *prev_prime;\n                    if (j == 0)\n                        prev_prime = rsa->p;\n                    else if (j == 1)\n                        prev_prime = rsa->q;\n                    else\n                        prev_prime = sk_RSA_PRIME_INFO_value(prime_infos,\n                                                             j - 2)->r;\n                    if (!BN_cmp(prime, prev_prime)) {\n                        goto redo;\n                    }\n                }\n            }\n            if (!BN_sub(r2, prime, BN_value_one()))\n                goto err;\n            ERR_set_mark();\n            BN_set_flags(r2, BN_FLG_CONSTTIME);\n            if (BN_mod_inverse(r1, r2, rsa->e, ctx) != NULL) {\n                break;\n            }\n            error = ERR_peek_last_error();\n            if (ERR_GET_LIB(error) == ERR_LIB_BN\n                && ERR_GET_REASON(error) == BN_R_NO_INVERSE) {\n                ERR_pop_to_mark();\n            } else {\n                goto err;\n            }\n            if (!BN_GENCB_call(cb, 2, n++))\n                goto err;\n        }\n        bitse += bitsr[i];\n        if (i == 1) {\n            if (!BN_mul(r1, rsa->p, rsa->q, ctx))\n                goto err;\n        } else if (i != 0) {\n            if (!BN_mul(r1, rsa->n, prime, ctx))\n                goto err;\n        } else {\n            if (!BN_GENCB_call(cb, 3, i))\n                goto err;\n            continue;\n        }\n        if (!BN_rshift(r2, r1, bitse - 4))\n            goto err;\n        bitst = BN_get_word(r2);\n        if (bitst < 0x9 || bitst > 0xF) {\n            bitse -= bitsr[i];\n            if (!BN_GENCB_call(cb, 2, n++))\n                goto err;\n            if (primes > 4) {\n                if (bitst < 0x9)\n                    adj++;\n                else\n                    adj--;\n            } else if (retries == 4) {\n                i = -1;\n                bitse = 0;\n                continue;\n            }\n            retries++;\n            goto redo;\n        }\n        if (i > 1 && BN_copy(pinfo->pp, rsa->n) == NULL)\n            goto err;\n        if (BN_copy(rsa->n, r1) == NULL)\n            goto err;\n        if (!BN_GENCB_call(cb, 3, i))\n            goto err;\n    }\n    if (BN_cmp(rsa->p, rsa->q) < 0) {\n        tmp = rsa->p;\n        rsa->p = rsa->q;\n        rsa->q = tmp;\n    }\n    if (!BN_sub(r1, rsa->p, BN_value_one()))\n        goto err;\n    if (!BN_sub(r2, rsa->q, BN_value_one()))\n        goto err;\n    if (!BN_mul(r0, r1, r2, ctx))\n        goto err;\n    for (i = 2; i < primes; i++) {\n        pinfo = sk_RSA_PRIME_INFO_value(prime_infos, i - 2);\n        if (!BN_sub(pinfo->d, pinfo->r, BN_value_one()))\n            goto err;\n        if (!BN_mul(r0, r0, pinfo->d, ctx))\n            goto err;\n    }\n    {\n        BIGNUM *pr0 = BN_new();\n        if (pr0 == NULL)\n            goto err;\n        BN_with_flags(pr0, r0, BN_FLG_CONSTTIME);\n        if (!BN_mod_inverse(rsa->d, rsa->e, pr0, ctx)) {\n            BN_free(pr0);\n            goto err;\n        }\n        BN_free(pr0);\n    }\n    {\n        BIGNUM *d = BN_new();\n        if (d == NULL)\n            goto err;\n        BN_with_flags(d, rsa->d, BN_FLG_CONSTTIME);\n        if (!BN_mod(rsa->dmp1, d, r1, ctx)\n            || !BN_mod(rsa->dmq1, d, r2, ctx)) {\n            BN_free(d);\n            goto err;\n        }\n        for (i = 2; i < primes; i++) {\n            pinfo = sk_RSA_PRIME_INFO_value(prime_infos, i - 2);\n            if (!BN_mod(pinfo->d, d, pinfo->d, ctx)) {\n                BN_free(d);\n                goto err;\n            }\n        }\n        BN_free(d);\n    }\n    {\n        BIGNUM *p = BN_new();\n        if (p == NULL)\n            goto err;\n        BN_with_flags(p, rsa->p, BN_FLG_CONSTTIME);\n        if (!BN_mod_inverse(rsa->iqmp, rsa->q, p, ctx)) {\n            BN_free(p);\n            goto err;\n        }\n        for (i = 2; i < primes; i++) {\n            pinfo = sk_RSA_PRIME_INFO_value(prime_infos, i - 2);\n            BN_with_flags(p, pinfo->r, BN_FLG_CONSTTIME);\n            if (!BN_mod_inverse(pinfo->t, pinfo->pp, p, ctx)) {\n                BN_free(p);\n                goto err;\n            }\n        }\n        BN_free(p);\n    }\n    ok = 1;\n err:\n    if (ok == -1) {\n        RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, ERR_LIB_BN);\n        ok = 0;\n    }\n    if (ctx != NULL)\n        BN_CTX_end(ctx);\n    BN_CTX_free(ctx);\n    return ok;\n#endif\n}', 'int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe,\n                         const BIGNUM *add, const BIGNUM *rem, BN_GENCB *cb)\n{\n    BIGNUM *t;\n    int found = 0;\n    int i, j, c1 = 0;\n    BN_CTX *ctx = NULL;\n    prime_t *mods = NULL;\n    int checks = BN_prime_checks_for_size(bits);\n    if (bits < 2) {\n        BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL);\n        return 0;\n    } else if (bits == 2 && safe) {\n        BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL);\n        return 0;\n    }\n    mods = OPENSSL_zalloc(sizeof(*mods) * NUMPRIMES);\n    if (mods == NULL)\n        goto err;\n    ctx = BN_CTX_new();\n    if (ctx == NULL)\n        goto err;\n    BN_CTX_start(ctx);\n    t = BN_CTX_get(ctx);\n    if (t == NULL)\n        goto err;\n loop:\n    if (add == NULL) {\n        if (!probable_prime(ret, bits, mods))\n            goto err;\n    } else {\n        if (safe) {\n            if (!probable_prime_dh_safe(ret, bits, add, rem, ctx))\n                goto err;\n        } else {\n            if (!bn_probable_prime_dh(ret, bits, add, rem, ctx))\n                goto err;\n        }\n    }\n    if (!BN_GENCB_call(cb, 0, c1++))\n        goto err;\n    if (!safe) {\n        i = BN_is_prime_fasttest_ex(ret, checks, ctx, 0, cb);\n        if (i == -1)\n            goto err;\n        if (i == 0)\n            goto loop;\n    } else {\n        if (!BN_rshift1(t, ret))\n            goto err;\n        for (i = 0; i < checks; i++) {\n            j = BN_is_prime_fasttest_ex(ret, 1, ctx, 0, cb);\n            if (j == -1)\n                goto err;\n            if (j == 0)\n                goto loop;\n            j = BN_is_prime_fasttest_ex(t, 1, ctx, 0, cb);\n            if (j == -1)\n                goto err;\n            if (j == 0)\n                goto loop;\n            if (!BN_GENCB_call(cb, 2, c1 - 1))\n                goto err;\n        }\n    }\n    found = 1;\n err:\n    OPENSSL_free(mods);\n    if (ctx != NULL)\n        BN_CTX_end(ctx);\n    BN_CTX_free(ctx);\n    bn_check_top(ret);\n    return found;\n}', 'static int probable_prime(BIGNUM *rnd, int bits, prime_t *mods)\n{\n    int i;\n    BN_ULONG delta;\n    BN_ULONG maxdelta = BN_MASK2 - primes[NUMPRIMES - 1];\n    char is_single_word = bits <= BN_BITS2;\n again:\n    if (!BN_priv_rand(rnd, bits, BN_RAND_TOP_TWO, BN_RAND_BOTTOM_ODD))\n        return 0;\n    for (i = 1; i < NUMPRIMES; i++) {\n        BN_ULONG mod = BN_mod_word(rnd, (BN_ULONG)primes[i]);\n        if (mod == (BN_ULONG)-1)\n            return 0;\n        mods[i] = (prime_t) mod;\n    }\n    if (is_single_word) {\n        BN_ULONG size_limit;\n        if (bits == BN_BITS2) {\n            size_limit = ~((BN_ULONG)0) - BN_get_word(rnd);\n        } else {\n            size_limit = (((BN_ULONG)1) << bits) - BN_get_word(rnd) - 1;\n        }\n        if (size_limit < maxdelta)\n            maxdelta = size_limit;\n    }\n    delta = 0;\n loop:\n    if (is_single_word) {\n        BN_ULONG rnd_word = BN_get_word(rnd);\n        for (i = 1; i < NUMPRIMES && primes[i] < rnd_word; i++) {\n            if ((mods[i] + delta) % primes[i] == 0) {\n                delta += 2;\n                if (delta > maxdelta)\n                    goto again;\n                goto loop;\n            }\n        }\n    } else {\n        for (i = 1; i < NUMPRIMES; i++) {\n            if (((mods[i] + delta) % primes[i]) <= 1) {\n                delta += 2;\n                if (delta > maxdelta)\n                    goto again;\n                goto loop;\n            }\n        }\n    }\n    if (!BN_add_word(rnd, delta))\n        return 0;\n    if (BN_num_bits(rnd) != bits)\n        goto again;\n    bn_check_top(rnd);\n    return 1;\n}', 'int BN_priv_rand(BIGNUM *rnd, int bits, int top, int bottom)\n{\n    return bnrand(PRIVATE, rnd, bits, top, bottom);\n}', 'static int bnrand(BNRAND_FLAG flag, BIGNUM *rnd, int bits, int top, int bottom)\n{\n    unsigned char *buf = NULL;\n    int b, ret = 0, bit, bytes, mask;\n    if (bits == 0) {\n        if (top != BN_RAND_TOP_ANY || bottom != BN_RAND_BOTTOM_ANY)\n            goto toosmall;\n        BN_zero(rnd);\n        return 1;\n    }\n    if (bits < 0 || (bits == 1 && top > 0))\n        goto toosmall;\n    bytes = (bits + 7) / 8;\n    bit = (bits - 1) % 8;\n    mask = 0xff << (bit + 1);\n    buf = OPENSSL_malloc(bytes);\n    if (buf == NULL) {\n        BNerr(BN_F_BNRAND, ERR_R_MALLOC_FAILURE);\n        goto err;\n    }\n    b = flag == NORMAL ? RAND_bytes(buf, bytes) : RAND_priv_bytes(buf, bytes);\n    if (b <= 0)\n        goto err;\n    if (flag == TESTING) {\n        int i;\n        unsigned char c;\n        for (i = 0; i < bytes; i++) {\n            if (RAND_bytes(&c, 1) <= 0)\n                goto err;\n            if (c >= 128 && i > 0)\n                buf[i] = buf[i - 1];\n            else if (c < 42)\n                buf[i] = 0;\n            else if (c < 84)\n                buf[i] = 255;\n        }\n    }\n    if (top >= 0) {\n        if (top) {\n            if (bit == 0) {\n                buf[0] = 1;\n                buf[1] |= 0x80;\n            } else {\n                buf[0] |= (3 << (bit - 1));\n            }\n        } else {\n            buf[0] |= (1 << bit);\n        }\n    }\n    buf[0] &= ~mask;\n    if (bottom)\n        buf[bytes - 1] |= 1;\n    if (!BN_bin2bn(buf, bytes, rnd))\n        goto err;\n    ret = 1;\n err:\n    OPENSSL_clear_free(buf, bytes);\n    bn_check_top(rnd);\n    return ret;\ntoosmall:\n    BNerr(BN_F_BNRAND, BN_R_BITS_TOO_SMALL);\n    return 0;\n}', 'void *CRYPTO_malloc(size_t num, const char *file, int line)\n{\n    void *ret = NULL;\n    INCREMENT(malloc_count);\n    if (malloc_impl != NULL && malloc_impl != CRYPTO_malloc)\n        return malloc_impl(num, file, line);\n    if (num == 0)\n        return NULL;\n    FAILTEST();\n    if (allow_customize) {\n        allow_customize = 0;\n    }\n#ifndef OPENSSL_NO_CRYPTO_MDEBUG\n    if (call_malloc_debug) {\n        CRYPTO_mem_debug_malloc(NULL, num, 0, file, line);\n        ret = malloc(num);\n        CRYPTO_mem_debug_malloc(ret, num, 1, file, line);\n    } else {\n        ret = malloc(num);\n    }\n#else\n    (void)(file); (void)(line);\n    ret = malloc(num);\n#endif\n    return ret;\n}']