crypto/rsa/rsa_gen.c:140: error: INTEGER_OVERFLOW_L2
  ([0, +oo] - 1):unsigned64 by call to `BN_cmp`.
Showing all 6 steps of the trace


crypto/rsa/rsa_gen.c:122:8: Call
120.   		if(!BN_generate_prime_ex(rsa->p, bitsp, 0, NULL, NULL, cb))
121.   			goto err;
122.   		if (!BN_sub(r2,rsa->p,BN_value_one())) goto err;
              ^
123.   		if (!BN_gcd(r1,r2,rsa->e,ctx)) goto err;
124.   		if (BN_is_one(r1)) break;

crypto/bn/bn_add.c:263:1: Parameter `a->top`
  261.   	}
  262.   
  263. > int BN_sub(BIGNUM *r, const BIGNUM *a, const BIGNUM *b)
  264.   	{
  265.   	size_t max;

crypto/rsa/rsa_gen.c:140:13: Call
138.   			if(!BN_generate_prime_ex(rsa->q, bitsq, 0, NULL, NULL, cb))
139.   				goto err;
140.   			} while((BN_cmp(rsa->p, rsa->q) == 0) && (++degenerate < 3));
                   ^
141.   		if(degenerate == 3)
142.   			{

crypto/bn/bn_lib.c:678:1: <LHS trace>
676.   	}
677.   
678. > int BN_cmp(const BIGNUM *a, const BIGNUM *b)
679.   	{
680.   	int i;

crypto/bn/bn_lib.c:678:1: Parameter `a->top`
  676.   	}
  677.   
  678. > int BN_cmp(const BIGNUM *a, const BIGNUM *b)
  679.   	{
  680.   	int i;

crypto/bn/bn_lib.c:709:7: Binary operation: ([0, +oo] - 1):unsigned64 by call to `BN_cmp` 
  707.   	if (a->top > b->top) return(gt);
  708.   	if (a->top < b->top) return(lt);
  709.   	for (i=a->top-1; i>=0; i--)
               ^
  710.   		{
  711.   		t1=a->d[i];

int BN_cmp(const BIGNUM *a, const BIGNUM *b)
	{
	int i;
	int gt,lt;
	BN_ULONG t1,t2;
	if ((a == NULL) || (b == NULL))
		{
		if (a != NULL)
			return(-1);
		else if (b != NULL)
			return(1);
		else
			return(0);
		}
	bn_check_top(a);
	bn_check_top(b);
	if (a->neg != b->neg)
		{
		if (a->neg)
			return(-1);
		else	return(1);
		}
	if (a->neg == 0)
		{ gt=1; lt= -1; }
	else	{ gt= -1; lt=1; }
	if (a->top > b->top) return(gt);
	if (a->top < b->top) return(lt);
	for (i=a->top-1; i>=0; i--)
		{
		t1=a->d[i];
		t2=b->d[i];
		if (t1 > t2) return(gt);
		if (t1 < t2) return(lt);
		}
	return(0);
	}
['static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb)\n\t{\n\tBIGNUM *r0=NULL,*r1=NULL,*r2=NULL,*r3=NULL,*tmp;\n\tBIGNUM local_r0,local_d,local_p;\n\tBIGNUM *pr0,*d,*p;\n\tint bitsp,bitsq,ok= -1,n=0;\n\tBN_CTX *ctx=NULL;\n\tctx=BN_CTX_new();\n\tif (ctx == NULL) goto err;\n\tBN_CTX_start(ctx);\n\tr0 = BN_CTX_get(ctx);\n\tr1 = BN_CTX_get(ctx);\n\tr2 = BN_CTX_get(ctx);\n\tr3 = BN_CTX_get(ctx);\n\tif (r3 == NULL) goto err;\n\tbitsp=(bits+1)/2;\n\tbitsq=bits-bitsp;\n\tif(!rsa->n && ((rsa->n=BN_new()) == NULL)) goto err;\n\tif(!rsa->d && ((rsa->d=BN_new()) == NULL)) goto err;\n\tif(!rsa->e && ((rsa->e=BN_new()) == NULL)) goto err;\n\tif(!rsa->p && ((rsa->p=BN_new()) == NULL)) goto err;\n\tif(!rsa->q && ((rsa->q=BN_new()) == NULL)) goto err;\n\tif(!rsa->dmp1 && ((rsa->dmp1=BN_new()) == NULL)) goto err;\n\tif(!rsa->dmq1 && ((rsa->dmq1=BN_new()) == NULL)) goto err;\n\tif(!rsa->iqmp && ((rsa->iqmp=BN_new()) == NULL)) goto err;\n\tBN_copy(rsa->e, e_value);\n\tfor (;;)\n\t\t{\n\t\tif(!BN_generate_prime_ex(rsa->p, bitsp, 0, NULL, NULL, cb))\n\t\t\tgoto err;\n\t\tif (!BN_sub(r2,rsa->p,BN_value_one())) goto err;\n\t\tif (!BN_gcd(r1,r2,rsa->e,ctx)) goto err;\n\t\tif (BN_is_one(r1)) break;\n\t\tif(!BN_GENCB_call(cb, 2, n++))\n\t\t\tgoto err;\n\t\t}\n\tif(!BN_GENCB_call(cb, 3, 0))\n\t\tgoto err;\n\tfor (;;)\n\t\t{\n\t\tunsigned int degenerate = 0;\n\t\tdo\n\t\t\t{\n\t\t\tif(!BN_generate_prime_ex(rsa->q, bitsq, 0, NULL, NULL, cb))\n\t\t\t\tgoto err;\n\t\t\t} while((BN_cmp(rsa->p, rsa->q) == 0) && (++degenerate < 3));\n\t\tif(degenerate == 3)\n\t\t\t{\n\t\t\tok = 0;\n\t\t\tRSAerr(RSA_F_RSA_BUILTIN_KEYGEN,RSA_R_KEY_SIZE_TOO_SMALL);\n\t\t\tgoto err;\n\t\t\t}\n\t\tif (!BN_sub(r2,rsa->q,BN_value_one())) goto err;\n\t\tif (!BN_gcd(r1,r2,rsa->e,ctx)) goto err;\n\t\tif (BN_is_one(r1))\n\t\t\tbreak;\n\t\tif(!BN_GENCB_call(cb, 2, n++))\n\t\t\tgoto err;\n\t\t}\n\tif(!BN_GENCB_call(cb, 3, 1))\n\t\tgoto err;\n\tif (BN_cmp(rsa->p,rsa->q) < 0)\n\t\t{\n\t\ttmp=rsa->p;\n\t\trsa->p=rsa->q;\n\t\trsa->q=tmp;\n\t\t}\n\tif (!BN_mul(rsa->n,rsa->p,rsa->q,ctx)) goto err;\n\tif (!BN_sub(r1,rsa->p,BN_value_one())) goto err;\n\tif (!BN_sub(r2,rsa->q,BN_value_one())) goto err;\n\tif (!BN_mul(r0,r1,r2,ctx)) goto err;\n\tif (!(rsa->flags & RSA_FLAG_NO_CONSTTIME))\n\t\t{\n\t\t  pr0 = &local_r0;\n\t\t  BN_with_flags(pr0, r0, BN_FLG_CONSTTIME);\n\t\t}\n\telse\n\t  pr0 = r0;\n\tif (!BN_mod_inverse(rsa->d,rsa->e,pr0,ctx)) goto err;\n\tif (!(rsa->flags & RSA_FLAG_NO_CONSTTIME))\n\t\t{\n\t\td = &local_d;\n\t\tBN_with_flags(d, rsa->d, BN_FLG_CONSTTIME);\n\t\t}\n\telse\n\t\td = rsa->d;\n\tif (!BN_mod(rsa->dmp1,d,r1,ctx)) goto err;\n\tif (!BN_mod(rsa->dmq1,d,r2,ctx)) goto err;\n\tif (!(rsa->flags & RSA_FLAG_NO_CONSTTIME))\n\t\t{\n\t\tp = &local_p;\n\t\tBN_with_flags(p, rsa->p, BN_FLG_CONSTTIME);\n\t\t}\n\telse\n\t\tp = rsa->p;\n\tif (!BN_mod_inverse(rsa->iqmp,rsa->q,p,ctx)) goto err;\n\tok=1;\nerr:\n\tif (ok == -1)\n\t\t{\n\t\tRSAerr(RSA_F_RSA_BUILTIN_KEYGEN,ERR_LIB_BN);\n\t\tok=0;\n\t\t}\n\tif (ctx != NULL)\n\t\t{\n\t\tBN_CTX_end(ctx);\n\t\tBN_CTX_free(ctx);\n\t\t}\n\treturn ok;\n\t}', 'int BN_sub(BIGNUM *r, const BIGNUM *a, const BIGNUM *b)\n\t{\n\tsize_t max;\n\tint add=0,neg=0;\n\tconst BIGNUM *tmp;\n\tbn_check_top(a);\n\tbn_check_top(b);\n\tif (a->neg)\n\t\t{\n\t\tif (b->neg)\n\t\t\t{ tmp=a; a=b; b=tmp; }\n\t\telse\n\t\t\t{ add=1; neg=1; }\n\t\t}\n\telse\n\t\t{\n\t\tif (b->neg) { add=1; neg=0; }\n\t\t}\n\tif (add)\n\t\t{\n\t\tif (!BN_uadd(r,a,b)) return(0);\n\t\tr->neg=neg;\n\t\treturn(1);\n\t\t}\n\tmax=(a->top > b->top)?a->top:b->top;\n\tif (bn_wexpand(r,max) == NULL) return(0);\n\tif (BN_ucmp(a,b) < 0)\n\t\t{\n\t\tif (!BN_usub(r,b,a)) return(0);\n\t\tr->neg=1;\n\t\t}\n\telse\n\t\t{\n\t\tif (!BN_usub(r,a,b)) return(0);\n\t\tr->neg=0;\n\t\t}\n\tbn_check_top(r);\n\treturn(1);\n\t}', 'int BN_cmp(const BIGNUM *a, const BIGNUM *b)\n\t{\n\tint i;\n\tint gt,lt;\n\tBN_ULONG t1,t2;\n\tif ((a == NULL) || (b == NULL))\n\t\t{\n\t\tif (a != NULL)\n\t\t\treturn(-1);\n\t\telse if (b != NULL)\n\t\t\treturn(1);\n\t\telse\n\t\t\treturn(0);\n\t\t}\n\tbn_check_top(a);\n\tbn_check_top(b);\n\tif (a->neg != b->neg)\n\t\t{\n\t\tif (a->neg)\n\t\t\treturn(-1);\n\t\telse\treturn(1);\n\t\t}\n\tif (a->neg == 0)\n\t\t{ gt=1; lt= -1; }\n\telse\t{ gt= -1; lt=1; }\n\tif (a->top > b->top) return(gt);\n\tif (a->top < b->top) return(lt);\n\tfor (i=a->top-1; i>=0; i--)\n\t\t{\n\t\tt1=a->d[i];\n\t\tt2=b->d[i];\n\t\tif (t1 > t2) return(gt);\n\t\tif (t1 < t2) return(lt);\n\t\t}\n\treturn(0);\n\t}']