crypto/bn/bn_gf2m.c:1102: error: BUFFER_OVERRUN_L3
  Offset: [-1, +oo] Size: [1, +oo] by call to `BN_GF2m_mod_solve_quad_arr`.
Showing all 16 steps of the trace


crypto/bn/bn_gf2m.c:1097:11: Call
1095.       if ((arr = OPENSSL_malloc(sizeof(*arr) * max)) == NULL)
1096.           goto err;
1097.       ret = BN_GF2m_poly2arr(p, arr, max);
                  ^
1098.       if (!ret || ret > max) {
1099.           BNerr(BN_F_BN_GF2M_MOD_SOLVE_QUAD, BN_R_INVALID_LENGTH);

crypto/bn/bn_gf2m.c:1116:1: Parameter `*p`
  1114.    * be filled if array was large enough.
  1115.    */
  1116. > int BN_GF2m_poly2arr(const BIGNUM *a, int p[], int max)
  1117.   {
  1118.       int i, j, k = 0;

crypto/bn/bn_gf2m.c:1102:11: Call
1100.           goto err;
1101.       }
1102.       ret = BN_GF2m_mod_solve_quad_arr(r, a, arr, ctx);
                  ^
1103.       bn_check_top(r);
1104.    err:

crypto/bn/bn_gf2m.c:983:1: Parameter `*p`
  981.    * 0. Uses algorithms A.4.7 and A.4.6 from IEEE P1363.
  982.    */
  983. > int BN_GF2m_mod_solve_quad_arr(BIGNUM *r, const BIGNUM *a_, const int p[],
  984.                                  BN_CTX *ctx)
  985.   {

crypto/bn/bn_gf2m.c:1034:18: Call
  1032.               goto err;
  1033.           do {
  1034.               if (!BN_priv_rand(rho, p[0], BN_RAND_TOP_ONE, BN_RAND_BOTTOM_ANY))
                           ^
  1035.                   goto err;
  1036.               if (!BN_GF2m_mod_arr(rho, rho, p))

crypto/bn/bn_rand.c:121:1: Parameter `bits`
    119.   }
    120.   
    121. > int BN_priv_rand(BIGNUM *rnd, int bits, int top, int bottom)
    122.   {
    123.       return bnrand(PRIVATE, rnd, bits, top, bottom, NULL);

crypto/bn/bn_rand.c:123:12: Call
    121.   int BN_priv_rand(BIGNUM *rnd, int bits, int top, int bottom)
    122.   {
    123.       return bnrand(PRIVATE, rnd, bits, top, bottom, NULL);
                      ^
    124.   }
    125.   

crypto/bn/bn_rand.c:62:14: <Offset trace>
60.           unsigned char c;
61.   
62.           for (i = 0; i < bytes; i++) {
                   ^
63.               if (rand_bytes_ex(libctx, &c, 1) <= 0)
64.                   goto err;

crypto/bn/bn_rand.c:62:14: Assignment
      60.           unsigned char c;
      61.   
      62.           for (i = 0; i < bytes; i++) {
                         ^
      63.               if (rand_bytes_ex(libctx, &c, 1) <= 0)
      64.                   goto err;

crypto/bn/bn_rand.c:23:1: <Length trace>
21.   } BNRAND_FLAG;
22.   
23. > static int bnrand(BNRAND_FLAG flag, BIGNUM *rnd, int bits, int top, int bottom,
24.                     BN_CTX *ctx)
25.   {

crypto/bn/bn_rand.c:23:1: Parameter `bits`
      21.   } BNRAND_FLAG;
      22.   
      23. > static int bnrand(BNRAND_FLAG flag, BIGNUM *rnd, int bits, int top, int bottom,
      24.                     BN_CTX *ctx)
      25.   {

crypto/bn/bn_rand.c:39:5: Assignment
      37.           goto toosmall;
      38.   
      39.       bytes = (bits + 7) / 8;
                ^
      40.       bit = (bits - 1) % 8;
      41.       mask = 0xff << (bit + 1);

crypto/bn/bn_rand.c:43:11: Call
      41.       mask = 0xff << (bit + 1);
      42.   
      43.       buf = OPENSSL_malloc(bytes);
                      ^
      44.       if (buf == NULL) {
      45.           BNerr(BN_F_BNRAND, ERR_R_MALLOC_FAILURE);

crypto/mem.c:201:9: Assignment
        199.   
        200.       if (num == 0)
        201.           return NULL;
                       ^
        202.   
        203.       FAILTEST();

crypto/bn/bn_rand.c:43:5: Assignment
      41.       mask = 0xff << (bit + 1);
      42.   
      43.       buf = OPENSSL_malloc(bytes);
                ^
      44.       if (buf == NULL) {
      45.           BNerr(BN_F_BNRAND, ERR_R_MALLOC_FAILURE);

crypto/bn/bn_rand.c:88:9: Array access: Offset: [-1, +oo] Size: [1, +oo] by call to `BN_GF2m_mod_solve_quad_arr` 
      86.       buf[0] &= ~mask;
      87.       if (bottom)                 /* set bottom bit if requested */
      88.           buf[bytes - 1] |= 1;
                    ^
      89.       if (!BN_bin2bn(buf, bytes, rnd))
      90.           goto err;

static int bnrand(BNRAND_FLAG flag, BIGNUM *rnd, int bits, int top, int bottom,
                  BN_CTX *ctx)
{
    unsigned char *buf = NULL;
    int b, ret = 0, bit, bytes, mask;
    OPENSSL_CTX *libctx = bn_get_lib_ctx(ctx);
    if (bits == 0) {
        if (top != BN_RAND_TOP_ANY || bottom != BN_RAND_BOTTOM_ANY)
            goto toosmall;
        BN_zero(rnd);
        return 1;
    }
    if (bits < 0 || (bits == 1 && top > 0))
        goto toosmall;
    bytes = (bits + 7) / 8;
    bit = (bits - 1) % 8;
    mask = 0xff << (bit + 1);
    buf = OPENSSL_malloc(bytes);
    if (buf == NULL) {
        BNerr(BN_F_BNRAND, ERR_R_MALLOC_FAILURE);
        goto err;
    }
    b = flag == NORMAL ? rand_bytes_ex(libctx, buf, bytes)
                       : rand_priv_bytes_ex(libctx, buf, bytes);
    if (b <= 0)
        goto err;
    if (flag == TESTING) {
        int i;
        unsigned char c;
        for (i = 0; i < bytes; i++) {
            if (rand_bytes_ex(libctx, &c, 1) <= 0)
                goto err;
            if (c >= 128 && i > 0)
                buf[i] = buf[i - 1];
            else if (c < 42)
                buf[i] = 0;
            else if (c < 84)
                buf[i] = 255;
        }
    }
    if (top >= 0) {
        if (top) {
            if (bit == 0) {
                buf[0] = 1;
                buf[1] |= 0x80;
            } else {
                buf[0] |= (3 << (bit - 1));
            }
        } else {
            buf[0] |= (1 << bit);
        }
    }
    buf[0] &= ~mask;
    if (bottom)
        buf[bytes - 1] |= 1;
    if (!BN_bin2bn(buf, bytes, rnd))
        goto err;
    ret = 1;
 err:
    OPENSSL_clear_free(buf, bytes);
    bn_check_top(rnd);
    return ret;
toosmall:
    BNerr(BN_F_BNRAND, BN_R_BITS_TOO_SMALL);
    return 0;
}
['int BN_GF2m_mod_solve_quad(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,\n                           BN_CTX *ctx)\n{\n    int ret = 0;\n    const int max = BN_num_bits(p) + 1;\n    int *arr = NULL;\n    bn_check_top(a);\n    bn_check_top(p);\n    if ((arr = OPENSSL_malloc(sizeof(*arr) * max)) == NULL)\n        goto err;\n    ret = BN_GF2m_poly2arr(p, arr, max);\n    if (!ret || ret > max) {\n        BNerr(BN_F_BN_GF2M_MOD_SOLVE_QUAD, BN_R_INVALID_LENGTH);\n        goto err;\n    }\n    ret = BN_GF2m_mod_solve_quad_arr(r, a, arr, ctx);\n    bn_check_top(r);\n err:\n    OPENSSL_free(arr);\n    return ret;\n}', 'int BN_GF2m_poly2arr(const BIGNUM *a, int p[], int max)\n{\n    int i, j, k = 0;\n    BN_ULONG mask;\n    if (BN_is_zero(a))\n        return 0;\n    for (i = a->top - 1; i >= 0; i--) {\n        if (!a->d[i])\n            continue;\n        mask = BN_TBIT;\n        for (j = BN_BITS2 - 1; j >= 0; j--) {\n            if (a->d[i] & mask) {\n                if (k < max)\n                    p[k] = BN_BITS2 * i + j;\n                k++;\n            }\n            mask >>= 1;\n        }\n    }\n    if (k < max) {\n        p[k] = -1;\n        k++;\n    }\n    return k;\n}', 'int BN_GF2m_mod_solve_quad_arr(BIGNUM *r, const BIGNUM *a_, const int p[],\n                               BN_CTX *ctx)\n{\n    int ret = 0, count = 0, j;\n    BIGNUM *a, *z, *rho, *w, *w2, *tmp;\n    bn_check_top(a_);\n    if (!p[0]) {\n        BN_zero(r);\n        return 1;\n    }\n    BN_CTX_start(ctx);\n    a = BN_CTX_get(ctx);\n    z = BN_CTX_get(ctx);\n    w = BN_CTX_get(ctx);\n    if (w == NULL)\n        goto err;\n    if (!BN_GF2m_mod_arr(a, a_, p))\n        goto err;\n    if (BN_is_zero(a)) {\n        BN_zero(r);\n        ret = 1;\n        goto err;\n    }\n    if (p[0] & 0x1) {\n        if (!BN_copy(z, a))\n            goto err;\n        for (j = 1; j <= (p[0] - 1) / 2; j++) {\n            if (!BN_GF2m_mod_sqr_arr(z, z, p, ctx))\n                goto err;\n            if (!BN_GF2m_mod_sqr_arr(z, z, p, ctx))\n                goto err;\n            if (!BN_GF2m_add(z, z, a))\n                goto err;\n        }\n    } else {\n        rho = BN_CTX_get(ctx);\n        w2 = BN_CTX_get(ctx);\n        tmp = BN_CTX_get(ctx);\n        if (tmp == NULL)\n            goto err;\n        do {\n            if (!BN_priv_rand(rho, p[0], BN_RAND_TOP_ONE, BN_RAND_BOTTOM_ANY))\n                goto err;\n            if (!BN_GF2m_mod_arr(rho, rho, p))\n                goto err;\n            BN_zero(z);\n            if (!BN_copy(w, rho))\n                goto err;\n            for (j = 1; j <= p[0] - 1; j++) {\n                if (!BN_GF2m_mod_sqr_arr(z, z, p, ctx))\n                    goto err;\n                if (!BN_GF2m_mod_sqr_arr(w2, w, p, ctx))\n                    goto err;\n                if (!BN_GF2m_mod_mul_arr(tmp, w2, a, p, ctx))\n                    goto err;\n                if (!BN_GF2m_add(z, z, tmp))\n                    goto err;\n                if (!BN_GF2m_add(w, w2, rho))\n                    goto err;\n            }\n            count++;\n        } while (BN_is_zero(w) && (count < MAX_ITERATIONS));\n        if (BN_is_zero(w)) {\n            BNerr(BN_F_BN_GF2M_MOD_SOLVE_QUAD_ARR, BN_R_TOO_MANY_ITERATIONS);\n            goto err;\n        }\n    }\n    if (!BN_GF2m_mod_sqr_arr(w, z, p, ctx))\n        goto err;\n    if (!BN_GF2m_add(w, z, w))\n        goto err;\n    if (BN_GF2m_cmp(w, a)) {\n        BNerr(BN_F_BN_GF2M_MOD_SOLVE_QUAD_ARR, BN_R_NO_SOLUTION);\n        goto err;\n    }\n    if (!BN_copy(r, z))\n        goto err;\n    bn_check_top(r);\n    ret = 1;\n err:\n    BN_CTX_end(ctx);\n    return ret;\n}', 'int BN_priv_rand(BIGNUM *rnd, int bits, int top, int bottom)\n{\n    return bnrand(PRIVATE, rnd, bits, top, bottom, NULL);\n}', 'static int bnrand(BNRAND_FLAG flag, BIGNUM *rnd, int bits, int top, int bottom,\n                  BN_CTX *ctx)\n{\n    unsigned char *buf = NULL;\n    int b, ret = 0, bit, bytes, mask;\n    OPENSSL_CTX *libctx = bn_get_lib_ctx(ctx);\n    if (bits == 0) {\n        if (top != BN_RAND_TOP_ANY || bottom != BN_RAND_BOTTOM_ANY)\n            goto toosmall;\n        BN_zero(rnd);\n        return 1;\n    }\n    if (bits < 0 || (bits == 1 && top > 0))\n        goto toosmall;\n    bytes = (bits + 7) / 8;\n    bit = (bits - 1) % 8;\n    mask = 0xff << (bit + 1);\n    buf = OPENSSL_malloc(bytes);\n    if (buf == NULL) {\n        BNerr(BN_F_BNRAND, ERR_R_MALLOC_FAILURE);\n        goto err;\n    }\n    b = flag == NORMAL ? rand_bytes_ex(libctx, buf, bytes)\n                       : rand_priv_bytes_ex(libctx, buf, bytes);\n    if (b <= 0)\n        goto err;\n    if (flag == TESTING) {\n        int i;\n        unsigned char c;\n        for (i = 0; i < bytes; i++) {\n            if (rand_bytes_ex(libctx, &c, 1) <= 0)\n                goto err;\n            if (c >= 128 && i > 0)\n                buf[i] = buf[i - 1];\n            else if (c < 42)\n                buf[i] = 0;\n            else if (c < 84)\n                buf[i] = 255;\n        }\n    }\n    if (top >= 0) {\n        if (top) {\n            if (bit == 0) {\n                buf[0] = 1;\n                buf[1] |= 0x80;\n            } else {\n                buf[0] |= (3 << (bit - 1));\n            }\n        } else {\n            buf[0] |= (1 << bit);\n        }\n    }\n    buf[0] &= ~mask;\n    if (bottom)\n        buf[bytes - 1] |= 1;\n    if (!BN_bin2bn(buf, bytes, rnd))\n        goto err;\n    ret = 1;\n err:\n    OPENSSL_clear_free(buf, bytes);\n    bn_check_top(rnd);\n    return ret;\ntoosmall:\n    BNerr(BN_F_BNRAND, BN_R_BITS_TOO_SMALL);\n    return 0;\n}', 'void *CRYPTO_malloc(size_t num, const char *file, int line)\n{\n    void *ret = NULL;\n    INCREMENT(malloc_count);\n    if (malloc_impl != NULL && malloc_impl != CRYPTO_malloc)\n        return malloc_impl(num, file, line);\n    if (num == 0)\n        return NULL;\n    FAILTEST();\n    if (allow_customize) {\n        allow_customize = 0;\n    }\n#if !defined(OPENSSL_NO_CRYPTO_MDEBUG) && !defined(FIPS_MODE)\n    if (call_malloc_debug) {\n        CRYPTO_mem_debug_malloc(NULL, num, 0, file, line);\n        ret = malloc(num);\n        CRYPTO_mem_debug_malloc(ret, num, 1, file, line);\n    } else {\n        ret = malloc(num);\n    }\n#else\n    (void)(file); (void)(line);\n    ret = malloc(num);\n#endif\n    return ret;\n}']