crypto/rsa/rsa_gen.c:152: error: BUFFER_OVERRUN_L3
  Offset: 0 Size: [0, 8388607] by call to `BN_generate_prime_ex`.
Showing all 13 steps of the trace


crypto/rsa/rsa_gen.c:88:1: Parameter `rsa->q->d`
86.   }
87.   
88. > static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value,
89.                                 BN_GENCB *cb)
90.   {

crypto/rsa/rsa_gen.c:152:18: Call
150.           unsigned int degenerate = 0;
151.           do {
152.               if (!BN_generate_prime_ex(rsa->q, bitsq, 0, NULL, NULL, cb))
                        ^
153.                   goto err;
154.           } while ((BN_cmp(rsa->p, rsa->q) == 0) && (++degenerate < 3));

crypto/bn/bn_prime.c:208:1: Parameter `*ret->d`
  206.   }
  207.   
  208. > int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe,
  209.                            const BIGNUM *add, const BIGNUM *rem, BN_GENCB *cb)
  210.   {

crypto/bn/bn_prime.c:241:14: Call
  239.       /* make a random number and set the top and bottom bits */
  240.       if (add == NULL) {
  241.           if (!probable_prime(ret, bits, mods))
                      ^
  242.               goto err;
  243.       } else {

crypto/bn/bn_prime.c:505:1: Parameter `*rnd->d`
    503.   }
    504.   
    505. > static int probable_prime(BIGNUM *rnd, int bits, prime_t *mods)
    506.   {
    507.       int i;

crypto/bn/bn_prime.c:513:10: Call
    511.   
    512.    again:
    513.       if (!BN_rand(rnd, bits, 1, 1))
                    ^
    514.           return (0);
    515.       /* we now have a random number 'rnd' to test. */

crypto/bn/bn_rand.c:200:1: Parameter `*rnd->d`
      198.   }
      199.   
      200. > int BN_rand(BIGNUM *rnd, int bits, int top, int bottom)
      201.   {
      202.       return bnrand(0, rnd, bits, top, bottom);

crypto/bn/bn_rand.c:202:12: Call
      200.   int BN_rand(BIGNUM *rnd, int bits, int top, int bottom)
      201.   {
      202.       return bnrand(0, rnd, bits, top, bottom);
                        ^
      203.   }
      204.   

crypto/bn/bn_rand.c:119:1: Parameter `*rnd->d`
        117.   #include <openssl/sha.h>
        118.   
        119. > static int bnrand(int pseudorand, BIGNUM *rnd, int bits, int top, int bottom)
        120.   {
        121.       unsigned char *buf = NULL;

crypto/bn/bn_rand.c:131:9: Call
        129.   
        130.       if (bits == 0) {
        131.           BN_zero(rnd);
                       ^
        132.           return 1;
        133.       }

crypto/bn/bn_lib.c:530:1: <Length trace>
528.   }
529.   
530. > int BN_set_word(BIGNUM *a, BN_ULONG w)
531.   {
532.       bn_check_top(a);

crypto/bn/bn_lib.c:530:1: Parameter `*a->d`
          528.   }
          529.   
          530. > int BN_set_word(BIGNUM *a, BN_ULONG w)
          531.   {
          532.       bn_check_top(a);

crypto/bn/bn_lib.c:536:5: Array access: Offset: 0 Size: [0, 8388607] by call to `BN_generate_prime_ex` 
          534.           return (0);
          535.       a->neg = 0;
          536.       a->d[0] = w;
                     ^
          537.       a->top = (w ? 1 : 0);
          538.       bn_check_top(a);

int BN_set_word(BIGNUM *a, BN_ULONG w)
{
    bn_check_top(a);
    if (bn_expand(a, (int)sizeof(BN_ULONG) * 8) == NULL)
        return (0);
    a->neg = 0;
    a->d[0] = w;
    a->top = (w ? 1 : 0);
    bn_check_top(a);
    return (1);
}
['static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value,\n                              BN_GENCB *cb)\n{\n    BIGNUM *r0 = NULL, *r1 = NULL, *r2 = NULL, *r3 = NULL, *tmp;\n    int bitsp, bitsq, ok = -1, n = 0;\n    BN_CTX *ctx = NULL;\n    ctx = BN_CTX_new();\n    if (ctx == NULL)\n        goto err;\n    BN_CTX_start(ctx);\n    r0 = BN_CTX_get(ctx);\n    r1 = BN_CTX_get(ctx);\n    r2 = BN_CTX_get(ctx);\n    r3 = BN_CTX_get(ctx);\n    if (r3 == NULL)\n        goto err;\n    bitsp = (bits + 1) / 2;\n    bitsq = bits - bitsp;\n    if (!rsa->n && ((rsa->n = BN_new()) == NULL))\n        goto err;\n    if (!rsa->d && ((rsa->d = BN_secure_new()) == NULL))\n        goto err;\n    if (!rsa->e && ((rsa->e = BN_new()) == NULL))\n        goto err;\n    if (!rsa->p && ((rsa->p = BN_secure_new()) == NULL))\n        goto err;\n    if (!rsa->q && ((rsa->q = BN_secure_new()) == NULL))\n        goto err;\n    if (!rsa->dmp1 && ((rsa->dmp1 = BN_secure_new()) == NULL))\n        goto err;\n    if (!rsa->dmq1 && ((rsa->dmq1 = BN_secure_new()) == NULL))\n        goto err;\n    if (!rsa->iqmp && ((rsa->iqmp = BN_secure_new()) == NULL))\n        goto err;\n    BN_copy(rsa->e, e_value);\n    for (;;) {\n        if (!BN_generate_prime_ex(rsa->p, bitsp, 0, NULL, NULL, cb))\n            goto err;\n        if (!BN_sub(r2, rsa->p, BN_value_one()))\n            goto err;\n        if (!BN_gcd(r1, r2, rsa->e, ctx))\n            goto err;\n        if (BN_is_one(r1))\n            break;\n        if (!BN_GENCB_call(cb, 2, n++))\n            goto err;\n    }\n    if (!BN_GENCB_call(cb, 3, 0))\n        goto err;\n    for (;;) {\n        unsigned int degenerate = 0;\n        do {\n            if (!BN_generate_prime_ex(rsa->q, bitsq, 0, NULL, NULL, cb))\n                goto err;\n        } while ((BN_cmp(rsa->p, rsa->q) == 0) && (++degenerate < 3));\n        if (degenerate == 3) {\n            ok = 0;\n            RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, RSA_R_KEY_SIZE_TOO_SMALL);\n            goto err;\n        }\n        if (!BN_sub(r2, rsa->q, BN_value_one()))\n            goto err;\n        if (!BN_gcd(r1, r2, rsa->e, ctx))\n            goto err;\n        if (BN_is_one(r1))\n            break;\n        if (!BN_GENCB_call(cb, 2, n++))\n            goto err;\n    }\n    if (!BN_GENCB_call(cb, 3, 1))\n        goto err;\n    if (BN_cmp(rsa->p, rsa->q) < 0) {\n        tmp = rsa->p;\n        rsa->p = rsa->q;\n        rsa->q = tmp;\n    }\n    if (!BN_mul(rsa->n, rsa->p, rsa->q, ctx))\n        goto err;\n    if (!BN_sub(r1, rsa->p, BN_value_one()))\n        goto err;\n    if (!BN_sub(r2, rsa->q, BN_value_one()))\n        goto err;\n    if (!BN_mul(r0, r1, r2, ctx))\n        goto err;\n    {\n        BIGNUM *local_r0 = NULL, *pr0;\n        if (!(rsa->flags & RSA_FLAG_NO_CONSTTIME)) {\n            pr0 = local_r0 = BN_new();\n            if (local_r0 == NULL)\n                goto err;\n            BN_with_flags(pr0, r0, BN_FLG_CONSTTIME);\n        } else {\n            pr0 = r0;\n        }\n        if (!BN_mod_inverse(rsa->d, rsa->e, pr0, ctx)) {\n            BN_free(local_r0);\n            goto err;\n        }\n        BN_free(local_r0);\n    }\n    {\n        BIGNUM *local_d = NULL, *d;\n        if (!(rsa->flags & RSA_FLAG_NO_CONSTTIME)) {\n            d = local_d = BN_new();\n            if (local_d == NULL)\n                goto err;\n            BN_with_flags(d, rsa->d, BN_FLG_CONSTTIME);\n        } else {\n            d = rsa->d;\n        }\n        if (\n               !BN_mod(rsa->dmp1, d, r1, ctx)\n            || !BN_mod(rsa->dmq1, d, r2, ctx)) {\n            BN_free(local_d);\n            goto err;\n        }\n        BN_free(local_d);\n    }\n    {\n        BIGNUM *local_p = NULL, *p;\n        if (!(rsa->flags & RSA_FLAG_NO_CONSTTIME)) {\n            p = local_p = BN_new();\n            if (local_p == NULL)\n                goto err;\n            BN_with_flags(p, rsa->p, BN_FLG_CONSTTIME);\n        } else {\n            p = rsa->p;\n        }\n        if (!BN_mod_inverse(rsa->iqmp, rsa->q, p, ctx)) {\n            BN_free(local_p);\n            goto err;\n        }\n        BN_free(local_p);\n    }\n    ok = 1;\n err:\n    if (ok == -1) {\n        RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, ERR_LIB_BN);\n        ok = 0;\n    }\n    if (ctx != NULL)\n        BN_CTX_end(ctx);\n    BN_CTX_free(ctx);\n    return ok;\n}', 'int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe,\n                         const BIGNUM *add, const BIGNUM *rem, BN_GENCB *cb)\n{\n    BIGNUM *t;\n    int found = 0;\n    int i, j, c1 = 0;\n    BN_CTX *ctx = NULL;\n    prime_t *mods = NULL;\n    int checks = BN_prime_checks_for_size(bits);\n    mods = OPENSSL_zalloc(sizeof(*mods) * NUMPRIMES);\n    if (mods == NULL)\n        goto err;\n    if (bits < 2) {\n        BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL);\n        return 0;\n    } else if (bits == 2 && safe) {\n        BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL);\n        return 0;\n    }\n    ctx = BN_CTX_new();\n    if (ctx == NULL)\n        goto err;\n    BN_CTX_start(ctx);\n    t = BN_CTX_get(ctx);\n    if (!t)\n        goto err;\n loop:\n    if (add == NULL) {\n        if (!probable_prime(ret, bits, mods))\n            goto err;\n    } else {\n        if (safe) {\n            if (!probable_prime_dh_safe(ret, bits, add, rem, ctx))\n                goto err;\n        } else {\n            if (!bn_probable_prime_dh(ret, bits, add, rem, ctx))\n                goto err;\n        }\n    }\n    if (!BN_GENCB_call(cb, 0, c1++))\n        goto err;\n    if (!safe) {\n        i = BN_is_prime_fasttest_ex(ret, checks, ctx, 0, cb);\n        if (i == -1)\n            goto err;\n        if (i == 0)\n            goto loop;\n    } else {\n        if (!BN_rshift1(t, ret))\n            goto err;\n        for (i = 0; i < checks; i++) {\n            j = BN_is_prime_fasttest_ex(ret, 1, ctx, 0, cb);\n            if (j == -1)\n                goto err;\n            if (j == 0)\n                goto loop;\n            j = BN_is_prime_fasttest_ex(t, 1, ctx, 0, cb);\n            if (j == -1)\n                goto err;\n            if (j == 0)\n                goto loop;\n            if (!BN_GENCB_call(cb, 2, c1 - 1))\n                goto err;\n        }\n    }\n    found = 1;\n err:\n    OPENSSL_free(mods);\n    if (ctx != NULL)\n        BN_CTX_end(ctx);\n    BN_CTX_free(ctx);\n    bn_check_top(ret);\n    return found;\n}', 'static int probable_prime(BIGNUM *rnd, int bits, prime_t *mods)\n{\n    int i;\n    BN_ULONG delta;\n    BN_ULONG maxdelta = BN_MASK2 - primes[NUMPRIMES - 1];\n    char is_single_word = bits <= BN_BITS2;\n again:\n    if (!BN_rand(rnd, bits, 1, 1))\n        return (0);\n    for (i = 1; i < NUMPRIMES; i++)\n        mods[i] = (prime_t) BN_mod_word(rnd, (BN_ULONG)primes[i]);\n    if (is_single_word) {\n        BN_ULONG size_limit;\n        if (bits == BN_BITS2) {\n            size_limit = ~((BN_ULONG)0) - BN_get_word(rnd);\n        } else {\n            size_limit = (((BN_ULONG)1) << bits) - BN_get_word(rnd) - 1;\n        }\n        if (size_limit < maxdelta)\n            maxdelta = size_limit;\n    }\n    delta = 0;\n loop:\n    if (is_single_word) {\n        BN_ULONG rnd_word = BN_get_word(rnd);\n        for (i = 1; i < NUMPRIMES && primes[i] < rnd_word; i++) {\n            if ((mods[i] + delta) % primes[i] == 0) {\n                delta += 2;\n                if (delta > maxdelta)\n                    goto again;\n                goto loop;\n            }\n        }\n    } else {\n        for (i = 1; i < NUMPRIMES; i++) {\n            if (((mods[i] + delta) % primes[i]) <= 1) {\n                delta += 2;\n                if (delta > maxdelta)\n                    goto again;\n                goto loop;\n            }\n        }\n    }\n    if (!BN_add_word(rnd, delta))\n        return (0);\n    if (BN_num_bits(rnd) != bits)\n        goto again;\n    bn_check_top(rnd);\n    return (1);\n}', 'int BN_rand(BIGNUM *rnd, int bits, int top, int bottom)\n{\n    return bnrand(0, rnd, bits, top, bottom);\n}', 'static int bnrand(int pseudorand, BIGNUM *rnd, int bits, int top, int bottom)\n{\n    unsigned char *buf = NULL;\n    int ret = 0, bit, bytes, mask;\n    time_t tim;\n    if (bits < 0 || (bits == 1 && top > 0)) {\n        BNerr(BN_F_BNRAND, BN_R_BITS_TOO_SMALL);\n        return 0;\n    }\n    if (bits == 0) {\n        BN_zero(rnd);\n        return 1;\n    }\n    bytes = (bits + 7) / 8;\n    bit = (bits - 1) % 8;\n    mask = 0xff << (bit + 1);\n    buf = OPENSSL_malloc(bytes);\n    if (buf == NULL) {\n        BNerr(BN_F_BNRAND, ERR_R_MALLOC_FAILURE);\n        goto err;\n    }\n    time(&tim);\n    RAND_add(&tim, sizeof(tim), 0.0);\n    if (pseudorand) {\n        if (RAND_bytes(buf, bytes) <= 0)\n            goto err;\n    } else {\n        if (RAND_bytes(buf, bytes) <= 0)\n            goto err;\n    }\n    if (pseudorand == 2) {\n        int i;\n        unsigned char c;\n        for (i = 0; i < bytes; i++) {\n            if (RAND_bytes(&c, 1) <= 0)\n                goto err;\n            if (c >= 128 && i > 0)\n                buf[i] = buf[i - 1];\n            else if (c < 42)\n                buf[i] = 0;\n            else if (c < 84)\n                buf[i] = 255;\n        }\n    }\n    if (top >= 0) {\n        if (top) {\n            if (bit == 0) {\n                buf[0] = 1;\n                buf[1] |= 0x80;\n            } else {\n                buf[0] |= (3 << (bit - 1));\n            }\n        } else {\n            buf[0] |= (1 << bit);\n        }\n    }\n    buf[0] &= ~mask;\n    if (bottom)\n        buf[bytes - 1] |= 1;\n    if (!BN_bin2bn(buf, bytes, rnd))\n        goto err;\n    ret = 1;\n err:\n    OPENSSL_clear_free(buf, bytes);\n    bn_check_top(rnd);\n    return (ret);\n}', 'int BN_set_word(BIGNUM *a, BN_ULONG w)\n{\n    bn_check_top(a);\n    if (bn_expand(a, (int)sizeof(BN_ULONG) * 8) == NULL)\n        return (0);\n    a->neg = 0;\n    a->d[0] = w;\n    a->top = (w ? 1 : 0);\n    bn_check_top(a);\n    return (1);\n}']