crypto/bn/bn_gf2m.c:1148: error: BUFFER_OVERRUN_L3
  Offset: [-1, +oo] Size: [1, +oo] by call to `BN_GF2m_mod_solve_quad_arr`.
Showing all 16 steps of the trace


crypto/bn/bn_gf2m.c:1143:11: Call
1141.       if ((arr = OPENSSL_malloc(sizeof(*arr) * max)) == NULL)
1142.           goto err;
1143.       ret = BN_GF2m_poly2arr(p, arr, max);
                  ^
1144.       if (!ret || ret > max) {
1145.           BNerr(BN_F_BN_GF2M_MOD_SOLVE_QUAD, BN_R_INVALID_LENGTH);

crypto/bn/bn_gf2m.c:1162:1: Parameter `*p`
  1160.    * be filled if array was large enough.
  1161.    */
  1162. > int BN_GF2m_poly2arr(const BIGNUM *a, int p[], int max)
  1163.   {
  1164.       int i, j, k = 0;

crypto/bn/bn_gf2m.c:1148:11: Call
1146.           goto err;
1147.       }
1148.       ret = BN_GF2m_mod_solve_quad_arr(r, a, arr, ctx);
                  ^
1149.       bn_check_top(r);
1150.    err:

crypto/bn/bn_gf2m.c:1029:1: Parameter `*p`
  1027.    * 0. Uses algorithms A.4.7 and A.4.6 from IEEE P1363.
  1028.    */
  1029. > int BN_GF2m_mod_solve_quad_arr(BIGNUM *r, const BIGNUM *a_, const int p[],
  1030.                                  BN_CTX *ctx)
  1031.   {

crypto/bn/bn_gf2m.c:1080:18: Call
  1078.               goto err;
  1079.           do {
  1080.               if (!BN_priv_rand(rho, p[0], BN_RAND_TOP_ONE, BN_RAND_BOTTOM_ANY))
                           ^
  1081.                   goto err;
  1082.               if (!BN_GF2m_mod_arr(rho, rho, p))

crypto/bn/bn_rand.c:107:1: Parameter `bits`
    105.   }
    106.   
    107. > int BN_priv_rand(BIGNUM *rnd, int bits, int top, int bottom)
    108.   {
    109.       return bnrand(PRIVATE, rnd, bits, top, bottom);

crypto/bn/bn_rand.c:109:12: Call
    107.   int BN_priv_rand(BIGNUM *rnd, int bits, int top, int bottom)
    108.   {
    109.       return bnrand(PRIVATE, rnd, bits, top, bottom);
                      ^
    110.   }
    111.   

crypto/bn/bn_rand.c:57:14: <Offset trace>
55.           unsigned char c;
56.   
57.           for (i = 0; i < bytes; i++) {
                   ^
58.               if (RAND_bytes(&c, 1) <= 0)
59.                   goto err;

crypto/bn/bn_rand.c:57:14: Assignment
      55.           unsigned char c;
      56.   
      57.           for (i = 0; i < bytes; i++) {
                         ^
      58.               if (RAND_bytes(&c, 1) <= 0)
      59.                   goto err;

crypto/bn/bn_rand.c:21:1: <Length trace>
19.   } BNRAND_FLAG;
20.   
21. > static int bnrand(BNRAND_FLAG flag, BIGNUM *rnd, int bits, int top, int bottom)
22.   {
23.       unsigned char *buf = NULL;

crypto/bn/bn_rand.c:21:1: Parameter `bits`
      19.   } BNRAND_FLAG;
      20.   
      21. > static int bnrand(BNRAND_FLAG flag, BIGNUM *rnd, int bits, int top, int bottom)
      22.   {
      23.       unsigned char *buf = NULL;

crypto/bn/bn_rand.c:35:5: Assignment
      33.           goto toosmall;
      34.   
      35.       bytes = (bits + 7) / 8;
                ^
      36.       bit = (bits - 1) % 8;
      37.       mask = 0xff << (bit + 1);

crypto/bn/bn_rand.c:39:11: Call
      37.       mask = 0xff << (bit + 1);
      38.   
      39.       buf = OPENSSL_malloc(bytes);
                      ^
      40.       if (buf == NULL) {
      41.           BNerr(BN_F_BNRAND, ERR_R_MALLOC_FAILURE);

crypto/mem.c:177:9: Assignment
        175.   
        176.       if (num == 0)
        177.           return NULL;
                       ^
        178.   
        179.       FAILTEST();

crypto/bn/bn_rand.c:39:5: Assignment
      37.       mask = 0xff << (bit + 1);
      38.   
      39.       buf = OPENSSL_malloc(bytes);
                ^
      40.       if (buf == NULL) {
      41.           BNerr(BN_F_BNRAND, ERR_R_MALLOC_FAILURE);

crypto/bn/bn_rand.c:83:9: Array access: Offset: [-1, +oo] Size: [1, +oo] by call to `BN_GF2m_mod_solve_quad_arr` 
      81.       buf[0] &= ~mask;
      82.       if (bottom)                 /* set bottom bit if requested */
      83.           buf[bytes - 1] |= 1;
                    ^
      84.       if (!BN_bin2bn(buf, bytes, rnd))
      85.           goto err;

static int bnrand(BNRAND_FLAG flag, BIGNUM *rnd, int bits, int top, int bottom)
{
    unsigned char *buf = NULL;
    int b, ret = 0, bit, bytes, mask;
    if (bits == 0) {
        if (top != BN_RAND_TOP_ANY || bottom != BN_RAND_BOTTOM_ANY)
            goto toosmall;
        BN_zero(rnd);
        return 1;
    }
    if (bits < 0 || (bits == 1 && top > 0))
        goto toosmall;
    bytes = (bits + 7) / 8;
    bit = (bits - 1) % 8;
    mask = 0xff << (bit + 1);
    buf = OPENSSL_malloc(bytes);
    if (buf == NULL) {
        BNerr(BN_F_BNRAND, ERR_R_MALLOC_FAILURE);
        goto err;
    }
    b = flag == NORMAL ? RAND_bytes(buf, bytes) : RAND_priv_bytes(buf, bytes);
    if (b <= 0)
        goto err;
    if (flag == TESTING) {
        int i;
        unsigned char c;
        for (i = 0; i < bytes; i++) {
            if (RAND_bytes(&c, 1) <= 0)
                goto err;
            if (c >= 128 && i > 0)
                buf[i] = buf[i - 1];
            else if (c < 42)
                buf[i] = 0;
            else if (c < 84)
                buf[i] = 255;
        }
    }
    if (top >= 0) {
        if (top) {
            if (bit == 0) {
                buf[0] = 1;
                buf[1] |= 0x80;
            } else {
                buf[0] |= (3 << (bit - 1));
            }
        } else {
            buf[0] |= (1 << bit);
        }
    }
    buf[0] &= ~mask;
    if (bottom)
        buf[bytes - 1] |= 1;
    if (!BN_bin2bn(buf, bytes, rnd))
        goto err;
    ret = 1;
 err:
    OPENSSL_clear_free(buf, bytes);
    bn_check_top(rnd);
    return (ret);
toosmall:
    BNerr(BN_F_BNRAND, BN_R_BITS_TOO_SMALL);
    return 0;
}
['int BN_GF2m_mod_solve_quad(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,\n                           BN_CTX *ctx)\n{\n    int ret = 0;\n    const int max = BN_num_bits(p) + 1;\n    int *arr = NULL;\n    bn_check_top(a);\n    bn_check_top(p);\n    if ((arr = OPENSSL_malloc(sizeof(*arr) * max)) == NULL)\n        goto err;\n    ret = BN_GF2m_poly2arr(p, arr, max);\n    if (!ret || ret > max) {\n        BNerr(BN_F_BN_GF2M_MOD_SOLVE_QUAD, BN_R_INVALID_LENGTH);\n        goto err;\n    }\n    ret = BN_GF2m_mod_solve_quad_arr(r, a, arr, ctx);\n    bn_check_top(r);\n err:\n    OPENSSL_free(arr);\n    return ret;\n}', 'int BN_GF2m_poly2arr(const BIGNUM *a, int p[], int max)\n{\n    int i, j, k = 0;\n    BN_ULONG mask;\n    if (BN_is_zero(a))\n        return 0;\n    for (i = a->top - 1; i >= 0; i--) {\n        if (!a->d[i])\n            continue;\n        mask = BN_TBIT;\n        for (j = BN_BITS2 - 1; j >= 0; j--) {\n            if (a->d[i] & mask) {\n                if (k < max)\n                    p[k] = BN_BITS2 * i + j;\n                k++;\n            }\n            mask >>= 1;\n        }\n    }\n    if (k < max) {\n        p[k] = -1;\n        k++;\n    }\n    return k;\n}', 'int BN_GF2m_mod_solve_quad_arr(BIGNUM *r, const BIGNUM *a_, const int p[],\n                               BN_CTX *ctx)\n{\n    int ret = 0, count = 0, j;\n    BIGNUM *a, *z, *rho, *w, *w2, *tmp;\n    bn_check_top(a_);\n    if (!p[0]) {\n        BN_zero(r);\n        return 1;\n    }\n    BN_CTX_start(ctx);\n    a = BN_CTX_get(ctx);\n    z = BN_CTX_get(ctx);\n    w = BN_CTX_get(ctx);\n    if (w == NULL)\n        goto err;\n    if (!BN_GF2m_mod_arr(a, a_, p))\n        goto err;\n    if (BN_is_zero(a)) {\n        BN_zero(r);\n        ret = 1;\n        goto err;\n    }\n    if (p[0] & 0x1) {\n        if (!BN_copy(z, a))\n            goto err;\n        for (j = 1; j <= (p[0] - 1) / 2; j++) {\n            if (!BN_GF2m_mod_sqr_arr(z, z, p, ctx))\n                goto err;\n            if (!BN_GF2m_mod_sqr_arr(z, z, p, ctx))\n                goto err;\n            if (!BN_GF2m_add(z, z, a))\n                goto err;\n        }\n    } else {\n        rho = BN_CTX_get(ctx);\n        w2 = BN_CTX_get(ctx);\n        tmp = BN_CTX_get(ctx);\n        if (tmp == NULL)\n            goto err;\n        do {\n            if (!BN_priv_rand(rho, p[0], BN_RAND_TOP_ONE, BN_RAND_BOTTOM_ANY))\n                goto err;\n            if (!BN_GF2m_mod_arr(rho, rho, p))\n                goto err;\n            BN_zero(z);\n            if (!BN_copy(w, rho))\n                goto err;\n            for (j = 1; j <= p[0] - 1; j++) {\n                if (!BN_GF2m_mod_sqr_arr(z, z, p, ctx))\n                    goto err;\n                if (!BN_GF2m_mod_sqr_arr(w2, w, p, ctx))\n                    goto err;\n                if (!BN_GF2m_mod_mul_arr(tmp, w2, a, p, ctx))\n                    goto err;\n                if (!BN_GF2m_add(z, z, tmp))\n                    goto err;\n                if (!BN_GF2m_add(w, w2, rho))\n                    goto err;\n            }\n            count++;\n        } while (BN_is_zero(w) && (count < MAX_ITERATIONS));\n        if (BN_is_zero(w)) {\n            BNerr(BN_F_BN_GF2M_MOD_SOLVE_QUAD_ARR, BN_R_TOO_MANY_ITERATIONS);\n            goto err;\n        }\n    }\n    if (!BN_GF2m_mod_sqr_arr(w, z, p, ctx))\n        goto err;\n    if (!BN_GF2m_add(w, z, w))\n        goto err;\n    if (BN_GF2m_cmp(w, a)) {\n        BNerr(BN_F_BN_GF2M_MOD_SOLVE_QUAD_ARR, BN_R_NO_SOLUTION);\n        goto err;\n    }\n    if (!BN_copy(r, z))\n        goto err;\n    bn_check_top(r);\n    ret = 1;\n err:\n    BN_CTX_end(ctx);\n    return ret;\n}', 'int BN_priv_rand(BIGNUM *rnd, int bits, int top, int bottom)\n{\n    return bnrand(PRIVATE, rnd, bits, top, bottom);\n}', 'static int bnrand(BNRAND_FLAG flag, BIGNUM *rnd, int bits, int top, int bottom)\n{\n    unsigned char *buf = NULL;\n    int b, ret = 0, bit, bytes, mask;\n    if (bits == 0) {\n        if (top != BN_RAND_TOP_ANY || bottom != BN_RAND_BOTTOM_ANY)\n            goto toosmall;\n        BN_zero(rnd);\n        return 1;\n    }\n    if (bits < 0 || (bits == 1 && top > 0))\n        goto toosmall;\n    bytes = (bits + 7) / 8;\n    bit = (bits - 1) % 8;\n    mask = 0xff << (bit + 1);\n    buf = OPENSSL_malloc(bytes);\n    if (buf == NULL) {\n        BNerr(BN_F_BNRAND, ERR_R_MALLOC_FAILURE);\n        goto err;\n    }\n    b = flag == NORMAL ? RAND_bytes(buf, bytes) : RAND_priv_bytes(buf, bytes);\n    if (b <= 0)\n        goto err;\n    if (flag == TESTING) {\n        int i;\n        unsigned char c;\n        for (i = 0; i < bytes; i++) {\n            if (RAND_bytes(&c, 1) <= 0)\n                goto err;\n            if (c >= 128 && i > 0)\n                buf[i] = buf[i - 1];\n            else if (c < 42)\n                buf[i] = 0;\n            else if (c < 84)\n                buf[i] = 255;\n        }\n    }\n    if (top >= 0) {\n        if (top) {\n            if (bit == 0) {\n                buf[0] = 1;\n                buf[1] |= 0x80;\n            } else {\n                buf[0] |= (3 << (bit - 1));\n            }\n        } else {\n            buf[0] |= (1 << bit);\n        }\n    }\n    buf[0] &= ~mask;\n    if (bottom)\n        buf[bytes - 1] |= 1;\n    if (!BN_bin2bn(buf, bytes, rnd))\n        goto err;\n    ret = 1;\n err:\n    OPENSSL_clear_free(buf, bytes);\n    bn_check_top(rnd);\n    return (ret);\ntoosmall:\n    BNerr(BN_F_BNRAND, BN_R_BITS_TOO_SMALL);\n    return 0;\n}', 'void *CRYPTO_malloc(size_t num, const char *file, int line)\n{\n    void *ret = NULL;\n    if (malloc_impl != NULL && malloc_impl != CRYPTO_malloc)\n        return malloc_impl(num, file, line);\n    if (num == 0)\n        return NULL;\n    FAILTEST();\n    allow_customize = 0;\n#ifndef OPENSSL_NO_CRYPTO_MDEBUG\n    if (call_malloc_debug) {\n        CRYPTO_mem_debug_malloc(NULL, num, 0, file, line);\n        ret = malloc(num);\n        CRYPTO_mem_debug_malloc(ret, num, 1, file, line);\n    } else {\n        ret = malloc(num);\n    }\n#else\n    osslargused(file); osslargused(line);\n    ret = malloc(num);\n#endif\n    return ret;\n}']