test/bntest.c:2286: error: BUFFER_OVERRUN_L3
  Offset: 1 Size: [1, 12] by call to `BN_generate_prime_ex`.
Showing all 15 steps of the trace


test/bntest.c:2277:1: Parameter `kBits`
2275.   }
2276.   
2277. > static int test_smallsafeprime(int kBits)
2278.   {
2279.       BIGNUM *r;

test/bntest.c:2286:14: Call
2284.   
2285.       if (kBits <= 5 && kBits != 3) {
2286.           if (!TEST_false(BN_generate_prime_ex(r, kBits, 1,
                     ^
2287.                                                NULL, NULL, NULL)))
2288.               goto err;

crypto/bn/bn_prime.c:87:1: Parameter `bits`
  85.   }
  86.   
  87. > int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe,
  88.                            const BIGNUM *add, const BIGNUM *rem, BN_GENCB *cb)
  89.   {

crypto/bn/bn_prime.c:125:14: Call
  123.       /* make a random number and set the top and bottom bits */
  124.       if (add == NULL) {
  125.           if (!probable_prime(ret, bits, mods))
                      ^
  126.               goto err;
  127.       } else {

crypto/bn/bn_prime.c:382:1: Parameter `bits`
    380.   }
    381.   
    382. > static int probable_prime(BIGNUM *rnd, int bits, prime_t *mods)
    383.   {
    384.       int i;

crypto/bn/bn_prime.c:391:10: Call
    389.    again:
    390.       /* TODO: Not all primes are private */
    391.       if (!BN_priv_rand(rnd, bits, BN_RAND_TOP_TWO, BN_RAND_BOTTOM_ODD))
                    ^
    392.           return 0;
    393.       /* we now have a random number 'rnd' to test. */

crypto/bn/bn_rand.c:121:1: Parameter `bits`
      119.   }
      120.   
      121. > int BN_priv_rand(BIGNUM *rnd, int bits, int top, int bottom)
      122.   {
      123.       return bnrand(PRIVATE, rnd, bits, top, bottom, NULL);

crypto/bn/bn_rand.c:123:12: Call
      121.   int BN_priv_rand(BIGNUM *rnd, int bits, int top, int bottom)
      122.   {
      123.       return bnrand(PRIVATE, rnd, bits, top, bottom, NULL);
                        ^
      124.   }
      125.   

crypto/bn/bn_rand.c:23:1: <Length trace>
21.   } BNRAND_FLAG;
22.   
23. > static int bnrand(BNRAND_FLAG flag, BIGNUM *rnd, int bits, int top, int bottom,
24.                     BN_CTX *ctx)
25.   {

crypto/bn/bn_rand.c:23:1: Parameter `bits`
        21.   } BNRAND_FLAG;
        22.   
        23. > static int bnrand(BNRAND_FLAG flag, BIGNUM *rnd, int bits, int top, int bottom,
        24.                     BN_CTX *ctx)
        25.   {

crypto/bn/bn_rand.c:39:5: Assignment
        37.           goto toosmall;
        38.   
        39.       bytes = (bits + 7) / 8;
                  ^
        40.       bit = (bits - 1) % 8;
        41.       mask = 0xff << (bit + 1);

crypto/bn/bn_rand.c:43:11: Call
        41.       mask = 0xff << (bit + 1);
        42.   
        43.       buf = OPENSSL_malloc(bytes);
                        ^
        44.       if (buf == NULL) {
        45.           BNerr(BN_F_BNRAND, ERR_R_MALLOC_FAILURE);

crypto/mem.c:201:9: Assignment
          199.   
          200.       if (num == 0)
          201.           return NULL;
                         ^
          202.   
          203.       FAILTEST();

crypto/bn/bn_rand.c:43:5: Assignment
        41.       mask = 0xff << (bit + 1);
        42.   
        43.       buf = OPENSSL_malloc(bytes);
                  ^
        44.       if (buf == NULL) {
        45.           BNerr(BN_F_BNRAND, ERR_R_MALLOC_FAILURE);

crypto/bn/bn_rand.c:78:17: Array access: Offset: 1 Size: [1, 12] by call to `BN_generate_prime_ex` 
        76.               if (bit == 0) {
        77.                   buf[0] = 1;
        78.                   buf[1] |= 0x80;
                              ^
        79.               } else {
        80.                   buf[0] |= (3 << (bit - 1));

static int bnrand(BNRAND_FLAG flag, BIGNUM *rnd, int bits, int top, int bottom,
                  BN_CTX *ctx)
{
    unsigned char *buf = NULL;
    int b, ret = 0, bit, bytes, mask;
    OPENSSL_CTX *libctx = bn_get_lib_ctx(ctx);
    if (bits == 0) {
        if (top != BN_RAND_TOP_ANY || bottom != BN_RAND_BOTTOM_ANY)
            goto toosmall;
        BN_zero(rnd);
        return 1;
    }
    if (bits < 0 || (bits == 1 && top > 0))
        goto toosmall;
    bytes = (bits + 7) / 8;
    bit = (bits - 1) % 8;
    mask = 0xff << (bit + 1);
    buf = OPENSSL_malloc(bytes);
    if (buf == NULL) {
        BNerr(BN_F_BNRAND, ERR_R_MALLOC_FAILURE);
        goto err;
    }
    b = flag == NORMAL ? rand_bytes_ex(libctx, buf, bytes)
                       : rand_priv_bytes_ex(libctx, buf, bytes);
    if (b <= 0)
        goto err;
    if (flag == TESTING) {
        int i;
        unsigned char c;
        for (i = 0; i < bytes; i++) {
            if (rand_bytes_ex(libctx, &c, 1) <= 0)
                goto err;
            if (c >= 128 && i > 0)
                buf[i] = buf[i - 1];
            else if (c < 42)
                buf[i] = 0;
            else if (c < 84)
                buf[i] = 255;
        }
    }
    if (top >= 0) {
        if (top) {
            if (bit == 0) {
                buf[0] = 1;
                buf[1] |= 0x80;
            } else {
                buf[0] |= (3 << (bit - 1));
            }
        } else {
            buf[0] |= (1 << bit);
        }
    }
    buf[0] &= ~mask;
    if (bottom)
        buf[bytes - 1] |= 1;
    if (!BN_bin2bn(buf, bytes, rnd))
        goto err;
    ret = 1;
 err:
    OPENSSL_clear_free(buf, bytes);
    bn_check_top(rnd);
    return ret;
toosmall:
    BNerr(BN_F_BNRAND, BN_R_BITS_TOO_SMALL);
    return 0;
}
['static int test_smallsafeprime(int kBits)\n{\n    BIGNUM *r;\n    int st = 0;\n    if (!TEST_ptr(r = BN_new()))\n        goto err;\n    if (kBits <= 5 && kBits != 3) {\n        if (!TEST_false(BN_generate_prime_ex(r, kBits, 1,\n                                             NULL, NULL, NULL)))\n            goto err;\n    } else {\n        if (!TEST_true(BN_generate_prime_ex(r, kBits, 1,\n                                            NULL, NULL, NULL))\n                || !TEST_int_eq(BN_num_bits(r), kBits))\n            goto err;\n    }\n    st = 1;\n err:\n    BN_free(r);\n    return st;\n}', 'int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe,\n                         const BIGNUM *add, const BIGNUM *rem, BN_GENCB *cb)\n{\n    BIGNUM *t;\n    int found = 0;\n    int i, j, c1 = 0;\n    BN_CTX *ctx = NULL;\n    prime_t *mods = NULL;\n    int checks = BN_prime_checks_for_size(bits);\n    if (bits < 2) {\n        BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL);\n        return 0;\n    } else if (add == NULL && safe && bits < 6 && bits != 3) {\n        BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL);\n        return 0;\n    }\n    mods = OPENSSL_zalloc(sizeof(*mods) * NUMPRIMES);\n    if (mods == NULL)\n        goto err;\n    ctx = BN_CTX_new();\n    if (ctx == NULL)\n        goto err;\n    BN_CTX_start(ctx);\n    t = BN_CTX_get(ctx);\n    if (t == NULL)\n        goto err;\n loop:\n    if (add == NULL) {\n        if (!probable_prime(ret, bits, mods))\n            goto err;\n    } else {\n        if (safe) {\n            if (!probable_prime_dh_safe(ret, bits, add, rem, ctx))\n                goto err;\n        } else {\n            if (!bn_probable_prime_dh(ret, bits, add, rem, ctx))\n                goto err;\n        }\n    }\n    if (!BN_GENCB_call(cb, 0, c1++))\n        goto err;\n    if (!safe) {\n        i = BN_is_prime_fasttest_ex(ret, checks, ctx, 0, cb);\n        if (i == -1)\n            goto err;\n        if (i == 0)\n            goto loop;\n    } else {\n        if (!BN_rshift1(t, ret))\n            goto err;\n        for (i = 0; i < checks; i++) {\n            j = BN_is_prime_fasttest_ex(ret, 1, ctx, 0, cb);\n            if (j == -1)\n                goto err;\n            if (j == 0)\n                goto loop;\n            j = BN_is_prime_fasttest_ex(t, 1, ctx, 0, cb);\n            if (j == -1)\n                goto err;\n            if (j == 0)\n                goto loop;\n            if (!BN_GENCB_call(cb, 2, c1 - 1))\n                goto err;\n        }\n    }\n    found = 1;\n err:\n    OPENSSL_free(mods);\n    BN_CTX_end(ctx);\n    BN_CTX_free(ctx);\n    bn_check_top(ret);\n    return found;\n}', 'static int probable_prime(BIGNUM *rnd, int bits, prime_t *mods)\n{\n    int i;\n    BN_ULONG delta;\n    BN_ULONG maxdelta = BN_MASK2 - primes[NUMPRIMES - 1];\n    char is_single_word = bits <= BN_BITS2;\n again:\n    if (!BN_priv_rand(rnd, bits, BN_RAND_TOP_TWO, BN_RAND_BOTTOM_ODD))\n        return 0;\n    for (i = 1; i < NUMPRIMES; i++) {\n        BN_ULONG mod = BN_mod_word(rnd, (BN_ULONG)primes[i]);\n        if (mod == (BN_ULONG)-1)\n            return 0;\n        mods[i] = (prime_t) mod;\n    }\n    if (is_single_word) {\n        BN_ULONG size_limit;\n        if (bits == BN_BITS2) {\n            size_limit = ~((BN_ULONG)0) - BN_get_word(rnd);\n        } else {\n            size_limit = (((BN_ULONG)1) << bits) - BN_get_word(rnd) - 1;\n        }\n        if (size_limit < maxdelta)\n            maxdelta = size_limit;\n    }\n    delta = 0;\n loop:\n    if (is_single_word) {\n        BN_ULONG rnd_word = BN_get_word(rnd);\n        for (i = 1; i < NUMPRIMES && primes[i] < rnd_word; i++) {\n            if ((mods[i] + delta) % primes[i] == 0) {\n                delta += 2;\n                if (delta > maxdelta)\n                    goto again;\n                goto loop;\n            }\n        }\n    } else {\n        for (i = 1; i < NUMPRIMES; i++) {\n            if (((mods[i] + delta) % primes[i]) <= 1) {\n                delta += 2;\n                if (delta > maxdelta)\n                    goto again;\n                goto loop;\n            }\n        }\n    }\n    if (!BN_add_word(rnd, delta))\n        return 0;\n    if (BN_num_bits(rnd) != bits)\n        goto again;\n    bn_check_top(rnd);\n    return 1;\n}', 'int BN_priv_rand(BIGNUM *rnd, int bits, int top, int bottom)\n{\n    return bnrand(PRIVATE, rnd, bits, top, bottom, NULL);\n}', 'static int bnrand(BNRAND_FLAG flag, BIGNUM *rnd, int bits, int top, int bottom,\n                  BN_CTX *ctx)\n{\n    unsigned char *buf = NULL;\n    int b, ret = 0, bit, bytes, mask;\n    OPENSSL_CTX *libctx = bn_get_lib_ctx(ctx);\n    if (bits == 0) {\n        if (top != BN_RAND_TOP_ANY || bottom != BN_RAND_BOTTOM_ANY)\n            goto toosmall;\n        BN_zero(rnd);\n        return 1;\n    }\n    if (bits < 0 || (bits == 1 && top > 0))\n        goto toosmall;\n    bytes = (bits + 7) / 8;\n    bit = (bits - 1) % 8;\n    mask = 0xff << (bit + 1);\n    buf = OPENSSL_malloc(bytes);\n    if (buf == NULL) {\n        BNerr(BN_F_BNRAND, ERR_R_MALLOC_FAILURE);\n        goto err;\n    }\n    b = flag == NORMAL ? rand_bytes_ex(libctx, buf, bytes)\n                       : rand_priv_bytes_ex(libctx, buf, bytes);\n    if (b <= 0)\n        goto err;\n    if (flag == TESTING) {\n        int i;\n        unsigned char c;\n        for (i = 0; i < bytes; i++) {\n            if (rand_bytes_ex(libctx, &c, 1) <= 0)\n                goto err;\n            if (c >= 128 && i > 0)\n                buf[i] = buf[i - 1];\n            else if (c < 42)\n                buf[i] = 0;\n            else if (c < 84)\n                buf[i] = 255;\n        }\n    }\n    if (top >= 0) {\n        if (top) {\n            if (bit == 0) {\n                buf[0] = 1;\n                buf[1] |= 0x80;\n            } else {\n                buf[0] |= (3 << (bit - 1));\n            }\n        } else {\n            buf[0] |= (1 << bit);\n        }\n    }\n    buf[0] &= ~mask;\n    if (bottom)\n        buf[bytes - 1] |= 1;\n    if (!BN_bin2bn(buf, bytes, rnd))\n        goto err;\n    ret = 1;\n err:\n    OPENSSL_clear_free(buf, bytes);\n    bn_check_top(rnd);\n    return ret;\ntoosmall:\n    BNerr(BN_F_BNRAND, BN_R_BITS_TOO_SMALL);\n    return 0;\n}', 'void *CRYPTO_malloc(size_t num, const char *file, int line)\n{\n    void *ret = NULL;\n    INCREMENT(malloc_count);\n    if (malloc_impl != NULL && malloc_impl != CRYPTO_malloc)\n        return malloc_impl(num, file, line);\n    if (num == 0)\n        return NULL;\n    FAILTEST();\n    if (allow_customize) {\n        allow_customize = 0;\n    }\n#if !defined(OPENSSL_NO_CRYPTO_MDEBUG) && !defined(FIPS_MODE)\n    if (call_malloc_debug) {\n        CRYPTO_mem_debug_malloc(NULL, num, 0, file, line);\n        ret = malloc(num);\n        CRYPTO_mem_debug_malloc(ret, num, 1, file, line);\n    } else {\n        ret = malloc(num);\n    }\n#else\n    (void)(file); (void)(line);\n    ret = malloc(num);\n#endif\n    return ret;\n}']