crypto/bn/bn_gf2m.c:959: error: INTEGER_OVERFLOW_L2
  ([0, +oo] - 1):unsigned32 by call to `BN_GF2m_mod_solve_quad_arr`.
Showing all 8 steps of the trace


crypto/bn/bn_gf2m.c:954:6: Call
952.   	unsigned int *arr=NULL, ret = 0;
953.   	if ((arr = (unsigned int *)OPENSSL_malloc(sizeof(unsigned int) * max)) == NULL) goto err;
954.   	if (BN_GF2m_poly2arr(p, arr, max) > max)
            ^
955.   		{
956.   		BNerr(BN_F_BN_GF2M_MOD_SOLVE_QUAD,BN_R_INVALID_LENGTH);

crypto/bn/bn_gf2m.c:971:1: Parameter `*p`
  969.    * number of coefficients that would be extracted if array was large enough.
  970.    */
  971. > int BN_GF2m_poly2arr(const BIGNUM *a, unsigned int p[], int max)
  972.   	{
  973.   	int i, j, k;

crypto/bn/bn_gf2m.c:959:8: Call
957.   		goto err;
958.   		}
959.   	ret = BN_GF2m_mod_solve_quad_arr(r, a, arr, ctx);
              ^
960.   	bn_check_top(r);
961.     err:

crypto/bn/bn_gf2m.c:868:1: <LHS trace>
866.    * Uses algorithms A.4.7 and A.4.6 from IEEE P1363.
867.    */
868. > int BN_GF2m_mod_solve_quad_arr(BIGNUM *r, const BIGNUM *a_, const unsigned int p[], BN_CTX *ctx)
869.   	{
870.   	int ret = 0, count = 0;

crypto/bn/bn_gf2m.c:868:1: Parameter `*p`
  866.    * Uses algorithms A.4.7 and A.4.6 from IEEE P1363.
  867.    */
  868. > int BN_GF2m_mod_solve_quad_arr(BIGNUM *r, const BIGNUM *a_, const unsigned int p[], BN_CTX *ctx)
  869.   	{
  870.   	int ret = 0, count = 0;

crypto/bn/bn_gf2m.c:880:7: Call
  878.   	if (w == NULL) goto err;
  879.   
  880.   	if (!BN_GF2m_mod_arr(a, a_, p)) goto err;
               ^
  881.   	
  882.   	if (BN_is_zero(a))

crypto/bn/bn_gf2m.c:320:1: Parameter `*p`
    318.   
    319.   /* Performs modular reduction of a and store result in r.  r could be a. */
    320. > int BN_GF2m_mod_arr(BIGNUM *r, const BIGNUM *a, const unsigned int p[])
    321.   	{
    322.   	int j, k;

crypto/bn/bn_gf2m.c:892:15: Binary operation: ([0, +oo] - 1):unsigned32 by call to `BN_GF2m_mod_solve_quad_arr` 
  890.   		/* compute half-trace of a */
  891.   		if (!BN_copy(z, a)) goto err;
  892.   		for (j = 1; j <= (p[0] - 1) / 2; j++)
                       ^
  893.   			{
  894.   			if (!BN_GF2m_mod_sqr_arr(z, z, p, ctx)) goto err;

int BN_GF2m_mod_solve_quad_arr(BIGNUM *r, const BIGNUM *a_, const unsigned int p[], BN_CTX *ctx)
	{
	int ret = 0, count = 0;
	unsigned int j;
	BIGNUM *a, *z, *rho, *w, *w2, *tmp;
	BN_CTX_start(ctx);
	a = BN_CTX_get(ctx);
	z = BN_CTX_get(ctx);
	w = BN_CTX_get(ctx);
	if (w == NULL) goto err;
	if (!BN_GF2m_mod_arr(a, a_, p)) goto err;
	if (BN_is_zero(a))
		{
		ret = BN_zero(r);
		goto err;
		}
	if (p[0] & 0x1)
		{
		if (!BN_copy(z, a)) goto err;
		for (j = 1; j <= (p[0] - 1) / 2; j++)
			{
			if (!BN_GF2m_mod_sqr_arr(z, z, p, ctx)) goto err;
			if (!BN_GF2m_mod_sqr_arr(z, z, p, ctx)) goto err;
			if (!BN_GF2m_add(z, z, a)) goto err;
			}
		}
	else
		{
		rho = BN_CTX_get(ctx);
		w2 = BN_CTX_get(ctx);
		tmp = BN_CTX_get(ctx);
		if (tmp == NULL) goto err;
		do
			{
			if (!BN_rand(rho, p[0], 0, 0)) goto err;
			if (!BN_GF2m_mod_arr(rho, rho, p)) goto err;
			if (!BN_zero(z)) goto err;
			if (!BN_copy(w, rho)) goto err;
			for (j = 1; j <= p[0] - 1; j++)
				{
				if (!BN_GF2m_mod_sqr_arr(z, z, p, ctx)) goto err;
				if (!BN_GF2m_mod_sqr_arr(w2, w, p, ctx)) goto err;
				if (!BN_GF2m_mod_mul_arr(tmp, w2, a, p, ctx)) goto err;
				if (!BN_GF2m_add(z, z, tmp)) goto err;
				if (!BN_GF2m_add(w, w2, rho)) goto err;
				}
			count++;
			} while (BN_is_zero(w) && (count < MAX_ITERATIONS));
		if (BN_is_zero(w))
			{
			BNerr(BN_F_BN_GF2M_MOD_SOLVE_QUAD_ARR,BN_R_TOO_MANY_ITERATIONS);
			goto err;
			}
		}
	if (!BN_GF2m_mod_sqr_arr(w, z, p, ctx)) goto err;
	if (!BN_GF2m_add(w, z, w)) goto err;
	if (BN_GF2m_cmp(w, a)) goto err;
	if (!BN_copy(r, z)) goto err;
	bn_check_top(r);
	ret = 1;
  err:
	BN_CTX_end(ctx);
	return ret;
	}
['int BN_GF2m_mod_solve_quad(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)\n\t{\n\tconst int max = BN_num_bits(p);\n\tunsigned int *arr=NULL, ret = 0;\n\tif ((arr = (unsigned int *)OPENSSL_malloc(sizeof(unsigned int) * max)) == NULL) goto err;\n\tif (BN_GF2m_poly2arr(p, arr, max) > max)\n\t\t{\n\t\tBNerr(BN_F_BN_GF2M_MOD_SOLVE_QUAD,BN_R_INVALID_LENGTH);\n\t\tgoto err;\n\t\t}\n\tret = BN_GF2m_mod_solve_quad_arr(r, a, arr, ctx);\n\tbn_check_top(r);\n  err:\n\tif (arr) OPENSSL_free(arr);\n\treturn ret;\n\t}', 'int BN_GF2m_poly2arr(const BIGNUM *a, unsigned int p[], int max)\n\t{\n\tint i, j, k;\n\tBN_ULONG mask;\n\tfor (k = 0; k < max; k++) p[k] = 0;\n\tk = 0;\n\tfor (i = a->top - 1; i >= 0; i--)\n\t\t{\n\t\tmask = BN_TBIT;\n\t\tfor (j = BN_BITS2 - 1; j >= 0; j--)\n\t\t\t{\n\t\t\tif (a->d[i] & mask)\n\t\t\t\t{\n\t\t\t\tif (k < max) p[k] = BN_BITS2 * i + j;\n\t\t\t\tk++;\n\t\t\t\t}\n\t\t\tmask >>= 1;\n\t\t\t}\n\t\t}\n\treturn k;\n\t}', 'int BN_GF2m_mod_solve_quad_arr(BIGNUM *r, const BIGNUM *a_, const unsigned int p[], BN_CTX *ctx)\n\t{\n\tint ret = 0, count = 0;\n\tunsigned int j;\n\tBIGNUM *a, *z, *rho, *w, *w2, *tmp;\n\tBN_CTX_start(ctx);\n\ta = BN_CTX_get(ctx);\n\tz = BN_CTX_get(ctx);\n\tw = BN_CTX_get(ctx);\n\tif (w == NULL) goto err;\n\tif (!BN_GF2m_mod_arr(a, a_, p)) goto err;\n\tif (BN_is_zero(a))\n\t\t{\n\t\tret = BN_zero(r);\n\t\tgoto err;\n\t\t}\n\tif (p[0] & 0x1)\n\t\t{\n\t\tif (!BN_copy(z, a)) goto err;\n\t\tfor (j = 1; j <= (p[0] - 1) / 2; j++)\n\t\t\t{\n\t\t\tif (!BN_GF2m_mod_sqr_arr(z, z, p, ctx)) goto err;\n\t\t\tif (!BN_GF2m_mod_sqr_arr(z, z, p, ctx)) goto err;\n\t\t\tif (!BN_GF2m_add(z, z, a)) goto err;\n\t\t\t}\n\t\t}\n\telse\n\t\t{\n\t\trho = BN_CTX_get(ctx);\n\t\tw2 = BN_CTX_get(ctx);\n\t\ttmp = BN_CTX_get(ctx);\n\t\tif (tmp == NULL) goto err;\n\t\tdo\n\t\t\t{\n\t\t\tif (!BN_rand(rho, p[0], 0, 0)) goto err;\n\t\t\tif (!BN_GF2m_mod_arr(rho, rho, p)) goto err;\n\t\t\tif (!BN_zero(z)) goto err;\n\t\t\tif (!BN_copy(w, rho)) goto err;\n\t\t\tfor (j = 1; j <= p[0] - 1; j++)\n\t\t\t\t{\n\t\t\t\tif (!BN_GF2m_mod_sqr_arr(z, z, p, ctx)) goto err;\n\t\t\t\tif (!BN_GF2m_mod_sqr_arr(w2, w, p, ctx)) goto err;\n\t\t\t\tif (!BN_GF2m_mod_mul_arr(tmp, w2, a, p, ctx)) goto err;\n\t\t\t\tif (!BN_GF2m_add(z, z, tmp)) goto err;\n\t\t\t\tif (!BN_GF2m_add(w, w2, rho)) goto err;\n\t\t\t\t}\n\t\t\tcount++;\n\t\t\t} while (BN_is_zero(w) && (count < MAX_ITERATIONS));\n\t\tif (BN_is_zero(w))\n\t\t\t{\n\t\t\tBNerr(BN_F_BN_GF2M_MOD_SOLVE_QUAD_ARR,BN_R_TOO_MANY_ITERATIONS);\n\t\t\tgoto err;\n\t\t\t}\n\t\t}\n\tif (!BN_GF2m_mod_sqr_arr(w, z, p, ctx)) goto err;\n\tif (!BN_GF2m_add(w, z, w)) goto err;\n\tif (BN_GF2m_cmp(w, a)) goto err;\n\tif (!BN_copy(r, z)) goto err;\n\tbn_check_top(r);\n\tret = 1;\n  err:\n\tBN_CTX_end(ctx);\n\treturn ret;\n\t}', 'int BN_GF2m_mod_arr(BIGNUM *r, const BIGNUM *a, const unsigned int p[])\n\t{\n\tint j, k;\n\tint n, dN, d0, d1;\n\tBN_ULONG zz, *z;\n\tif (a != r)\n\t\t{\n\t\tif (!bn_wexpand(r, a->top)) return 0;\n\t\tfor (j = 0; j < a->top; j++)\n\t\t\t{\n\t\t\tr->d[j] = a->d[j];\n\t\t\t}\n\t\tr->top = a->top;\n\t\t}\n\tz = r->d;\n\tdN = p[0] / BN_BITS2;\n\tfor (j = r->top - 1; j > dN;)\n\t\t{\n\t\tzz = z[j];\n\t\tif (z[j] == 0) { j--; continue; }\n\t\tz[j] = 0;\n\t\tfor (k = 1; p[k] > 0; k++)\n\t\t\t{\n\t\t\tn = p[0] - p[k];\n\t\t\td0 = n % BN_BITS2;  d1 = BN_BITS2 - d0;\n\t\t\tn /= BN_BITS2;\n\t\t\tz[j-n] ^= (zz>>d0);\n\t\t\tif (d0) z[j-n-1] ^= (zz<<d1);\n\t\t\t}\n\t\tn = dN;\n\t\td0 = p[0] % BN_BITS2;\n\t\td1 = BN_BITS2 - d0;\n\t\tz[j-n] ^= (zz >> d0);\n\t\tif (d0) z[j-n-1] ^= (zz << d1);\n\t\t}\n\twhile (j == dN)\n\t\t{\n\t\td0 = p[0] % BN_BITS2;\n\t\tzz = z[dN] >> d0;\n\t\tif (zz == 0) break;\n\t\td1 = BN_BITS2 - d0;\n\t\tif (d0) z[dN] = (z[dN] << d1) >> d1;\n\t\tz[0] ^= zz;\n\t\tfor (k = 1; p[k] > 0; k++)\n\t\t\t{\n\t\t\tBN_ULONG tmp_ulong;\n\t\t\tn = p[k] / BN_BITS2;\n\t\t\td0 = p[k] % BN_BITS2;\n\t\t\td1 = BN_BITS2 - d0;\n\t\t\tz[n] ^= (zz << d0);\n\t\t\ttmp_ulong = zz >> d1;\n                        if (d0 && tmp_ulong)\n                                z[n+1] ^= tmp_ulong;\n\t\t\t}\n\t\t}\n\tbn_correct_top(r);\n\treturn 1;\n\t}']