crypto/rsa/rsa_chk.c:122: error: BUFFER_OVERRUN_L3
  Offset: [-1, +oo] Size: 13 by call to `BN_mod_mul`.
Showing all 16 steps of the trace


crypto/rsa/rsa_chk.c:68:8: Call
66.   	l = BN_new();
67.   	m = BN_new();
68.   	ctx = BN_CTX_new();
             ^
69.   	if (i == NULL || j == NULL || k == NULL || l == NULL ||
70.   		m == NULL || ctx == NULL)

crypto/bn/bn_lib.c:319:2: Call
  317.   		}
  318.   
  319.   	BN_CTX_init(ret);
          ^
  320.   	ret->flags=BN_FLG_MALLOCED;
  321.   	return(ret);

crypto/bn/bn_lib.c:327:2: Assignment
    325.   	{
    326.   	memset(ctx,0,sizeof(BN_CTX));
    327.   	ctx->tos=0;
            ^
    328.   	ctx->flags=0;
    329.   	}

crypto/rsa/rsa_chk.c:78:6: Call
76.   	
77.   	/* p prime? */
78.   	r = BN_is_prime(key->p, BN_prime_checks, NULL, NULL, NULL);
           ^
79.   	if (r != 1)
80.   		{

crypto/bn/bn_prime.c:154:1: Parameter `ctx_passed->tos`
  152.   	}
  153.   
  154. > int BN_is_prime(BIGNUM *a, int checks, void (*callback)(int,int,void *),
  155.   	     BN_CTX *ctx_passed, void *cb_arg)
  156.   	{

crypto/rsa/rsa_chk.c:88:6: Call
86.   	
87.   	/* q prime? */
88.   	r = BN_is_prime(key->q, BN_prime_checks, NULL, NULL, NULL);
           ^
89.   	if (r != 1)
90.   		{

crypto/bn/bn_prime.c:154:1: Parameter `ctx_passed->tos`
  152.   	}
  153.   
  154. > int BN_is_prime(BIGNUM *a, int checks, void (*callback)(int,int,void *),
  155.   	     BN_CTX *ctx_passed, void *cb_arg)
  156.   	{

crypto/rsa/rsa_chk.c:122:6: Call
120.   	if (!r) { ret = -1; goto err; }
121.   
122.   	r = BN_mod_mul(i, key->d, key->e, k, ctx);
            ^
123.   	if (!r) { ret = -1; goto err; }
124.   

crypto/bn/bn_exp.c:66:1: Parameter `ctx->tos`
  64.   
  65.   /* slow but works */
  66. > int BN_mod_mul(BIGNUM *ret, BIGNUM *a, BIGNUM *b, const BIGNUM *m, BN_CTX *ctx)
  67.   	{
  68.   	BIGNUM *t;

crypto/bn/bn_exp.c:75:15: Assignment
  73.   	bn_check_top(m);
  74.   
  75.   	t= &(ctx->bn[ctx->tos++]);
                      ^
  76.   	if (a == b)
  77.   		{ if (!BN_sqr(t,a,ctx)) goto err; }

crypto/bn/bn_exp.c:77:10: Call
  75.   	t= &(ctx->bn[ctx->tos++]);
  76.   	if (a == b)
  77.   		{ if (!BN_sqr(t,a,ctx)) goto err; }
                 ^
  78.   	else
  79.   		{ if (!BN_mul(t,a,b,ctx)) goto err; }

crypto/bn/bn_sqr.c:65:1: <Offset trace>
63.   /* r must not be a */
64.   /* I've just gone over this and it is now %20 faster on x86 - eay - 27 Jun 96 */
65. > int BN_sqr(BIGNUM *r, BIGNUM *a, BN_CTX *ctx)
66.   	{
67.   	int max,al;

crypto/bn/bn_sqr.c:65:1: Parameter `ctx->tos`
    63.   /* r must not be a */
    64.   /* I've just gone over this and it is now %20 faster on x86 - eay - 27 Jun 96 */
    65. > int BN_sqr(BIGNUM *r, BIGNUM *a, BN_CTX *ctx)
    66.   	{
    67.   	int max,al;

crypto/bn/bn_sqr.c:65:1: <Length trace>
63.   /* r must not be a */
64.   /* I've just gone over this and it is now %20 faster on x86 - eay - 27 Jun 96 */
65. > int BN_sqr(BIGNUM *r, BIGNUM *a, BN_CTX *ctx)
66.   	{
67.   	int max,al;

crypto/bn/bn_sqr.c:65:1: Parameter `ctx->bn[*]`
    63.   /* r must not be a */
    64.   /* I've just gone over this and it is now %20 faster on x86 - eay - 27 Jun 96 */
    65. > int BN_sqr(BIGNUM *r, BIGNUM *a, BN_CTX *ctx)
    66.   	{
    67.   	int max,al;

crypto/bn/bn_sqr.c:74:2: Array access: Offset: [-1, +oo] Size: 13 by call to `BN_mod_mul` 
    72.   #endif
    73.   	bn_check_top(a);
    74.   	tmp= &(ctx->bn[ctx->tos]);
           ^
    75.   	rr=(a != r)?r: (&ctx->bn[ctx->tos+1]);
    76.   

int BN_sqr(BIGNUM *r, BIGNUM *a, BN_CTX *ctx)
	{
	int max,al;
	BIGNUM *tmp,*rr;
#ifdef BN_COUNT
printf("BN_sqr %d * %d\n",a->top,a->top);
#endif
	bn_check_top(a);
	tmp= &(ctx->bn[ctx->tos]);
	rr=(a != r)?r: (&ctx->bn[ctx->tos+1]);
	al=a->top;
	if (al <= 0)
		{
		r->top=0;
		return(1);
		}
	max=(al+al);
	if (bn_wexpand(rr,max+1) == NULL) return(0);
	r->neg=0;
	if (al == 4)
		{
#ifndef BN_SQR_COMBA
		BN_ULONG t[8];
		bn_sqr_normal(rr->d,a->d,4,t);
#else
		bn_sqr_comba4(rr->d,a->d);
#endif
		}
	else if (al == 8)
		{
#ifndef BN_SQR_COMBA
		BN_ULONG t[16];
		bn_sqr_normal(rr->d,a->d,8,t);
#else
		bn_sqr_comba8(rr->d,a->d);
#endif
		}
	else
		{
#if defined(BN_RECURSION)
		if (al < BN_SQR_RECURSIVE_SIZE_NORMAL)
			{
			BN_ULONG t[BN_SQR_RECURSIVE_SIZE_NORMAL*2];
			bn_sqr_normal(rr->d,a->d,al,t);
			}
		else
			{
			int j,k;
			j=BN_num_bits_word((BN_ULONG)al);
			j=1<<(j-1);
			k=j+j;
			if (al == j)
				{
				if (bn_wexpand(a,k*2) == NULL) return(0);
				if (bn_wexpand(tmp,k*2) == NULL) return(0);
				bn_sqr_recursive(rr->d,a->d,al,tmp->d);
				}
			else
				{
				if (bn_wexpand(tmp,max) == NULL) return(0);
				bn_sqr_normal(rr->d,a->d,al,tmp->d);
				}
			}
#else
		if (bn_wexpand(tmp,max) == NULL) return(0);
		bn_sqr_normal(rr->d,a->d,al,tmp->d);
#endif
		}
	rr->top=max;
	if ((max > 0) && (rr->d[max-1] == 0)) rr->top--;
	if (rr != r) BN_copy(r,rr);
	return(1);
	}
['int RSA_check_key(RSA *key)\n\t{\n\tBIGNUM *i, *j, *k, *l, *m;\n\tBN_CTX *ctx;\n\tint r;\n\tint ret=1;\n\ti = BN_new();\n\tj = BN_new();\n\tk = BN_new();\n\tl = BN_new();\n\tm = BN_new();\n\tctx = BN_CTX_new();\n\tif (i == NULL || j == NULL || k == NULL || l == NULL ||\n\t\tm == NULL || ctx == NULL)\n\t\t{\n\t\tret = -1;\n\t\tRSAerr(RSA_F_RSA_CHECK_KEY, ERR_R_MALLOC_FAILURE);\n\t\tgoto err;\n\t\t}\n\tr = BN_is_prime(key->p, BN_prime_checks, NULL, NULL, NULL);\n\tif (r != 1)\n\t\t{\n\t\tret = r;\n\t\tif (r != 0)\n\t\t\tgoto err;\n\t\tRSAerr(RSA_F_RSA_CHECK_KEY, RSA_R_P_NOT_PRIME);\n\t\t}\n\tr = BN_is_prime(key->q, BN_prime_checks, NULL, NULL, NULL);\n\tif (r != 1)\n\t\t{\n\t\tret = r;\n\t\tif (r != 0)\n\t\t\tgoto err;\n\t\tRSAerr(RSA_F_RSA_CHECK_KEY, RSA_R_Q_NOT_PRIME);\n\t\t}\n\tr = BN_mul(i, key->p, key->q, ctx);\n\tif (!r) { ret = -1; goto err; }\n\tif (BN_cmp(i, key->n) != 0)\n\t\t{\n\t\tret = 0;\n\t\tRSAerr(RSA_F_RSA_CHECK_KEY, RSA_R_N_DOES_NOT_EQUAL_P_Q);\n\t\t}\n\tr = BN_sub(i, key->p, BN_value_one());\n\tif (!r) { ret = -1; goto err; }\n\tr = BN_sub(j, key->q, BN_value_one());\n\tif (!r) { ret = -1; goto err; }\n\tr = BN_mul(l, i, j, ctx);\n\tif (!r) { ret = -1; goto err; }\n\tr = BN_gcd(m, i, j, ctx);\n\tif (!r) { ret = -1; goto err; }\n\tr = BN_div(k, NULL, l, m, ctx);\n\tif (!r) { ret = -1; goto err; }\n\tr = BN_mod_mul(i, key->d, key->e, k, ctx);\n\tif (!r) { ret = -1; goto err; }\n\tif (!BN_is_one(i))\n\t\t{\n\t\tret = 0;\n\t\tRSAerr(RSA_F_RSA_CHECK_KEY, RSA_R_D_E_NOT_CONGRUENT_TO_1);\n\t\t}\n\tif (key->dmp1 != NULL && key->dmq1 != NULL && key->iqmp != NULL)\n\t\t{\n\t\tr = BN_sub(i, key->p, BN_value_one());\n\t\tif (!r) { ret = -1; goto err; }\n\t\tr = BN_mod(j, key->d, i, ctx);\n\t\tif (!r) { ret = -1; goto err; }\n\t\tif (BN_cmp(j, key->dmp1) != 0)\n\t\t\t{\n\t\t\tret = 0;\n\t\t\tRSAerr(RSA_F_RSA_CHECK_KEY,\n\t\t\t\tRSA_R_DMP1_NOT_CONGRUENT_TO_D);\n\t\t\t}\n\t\tr = BN_sub(i, key->q, BN_value_one());\n\t\tif (!r) { ret = -1; goto err; }\n\t\tr = BN_mod(j, key->d, i, ctx);\n\t\tif (!r) { ret = -1; goto err; }\n\t\tif (BN_cmp(j, key->dmq1) != 0)\n\t\t\t{\n\t\t\tret = 0;\n\t\t\tRSAerr(RSA_F_RSA_CHECK_KEY,\n\t\t\t\tRSA_R_DMQ1_NOT_CONGRUENT_TO_D);\n\t\t\t}\n\t\tif(!BN_mod_inverse(i, key->q, key->p, ctx))\n\t\t\t{\n\t\t\tret = -1;\n\t\t\tgoto err;\n\t\t\t}\n\t\tif (BN_cmp(i, key->iqmp) != 0)\n\t\t\t{\n\t\t\tret = 0;\n\t\t\tRSAerr(RSA_F_RSA_CHECK_KEY,\n\t\t\t\tRSA_R_IQMP_NOT_INVERSE_OF_Q);\n\t\t\t}\n\t\t}\n err:\n\tif (i != NULL) BN_free(i);\n\tif (j != NULL) BN_free(j);\n\tif (k != NULL) BN_free(k);\n\tif (l != NULL) BN_free(l);\n\tif (m != NULL) BN_free(m);\n\tif (ctx != NULL) BN_CTX_free(ctx);\n\treturn (ret);\n\t}', 'BN_CTX *BN_CTX_new(void)\n\t{\n\tBN_CTX *ret;\n\tret=(BN_CTX *)Malloc(sizeof(BN_CTX));\n\tif (ret == NULL)\n\t\t{\n\t\tBNerr(BN_F_BN_CTX_NEW,ERR_R_MALLOC_FAILURE);\n\t\treturn(NULL);\n\t\t}\n\tBN_CTX_init(ret);\n\tret->flags=BN_FLG_MALLOCED;\n\treturn(ret);\n\t}', 'void BN_CTX_init(BN_CTX *ctx)\n\t{\n\tmemset(ctx,0,sizeof(BN_CTX));\n\tctx->tos=0;\n\tctx->flags=0;\n\t}', 'int BN_is_prime(BIGNUM *a, int checks, void (*callback)(int,int,void *),\n\t     BN_CTX *ctx_passed, void *cb_arg)\n\t{\n\tint i,j,c2=0,ret= -1;\n\tBIGNUM *check;\n\tBN_CTX *ctx=NULL,*ctx2=NULL;\n\tBN_MONT_CTX *mont=NULL;\n\tif (!BN_is_odd(a))\n\t\treturn(0);\n\tif (ctx_passed != NULL)\n\t\tctx=ctx_passed;\n\telse\n\t\tif ((ctx=BN_CTX_new()) == NULL) goto err;\n\tif ((ctx2=BN_CTX_new()) == NULL) goto err;\n\tif ((mont=BN_MONT_CTX_new()) == NULL) goto err;\n\tcheck= &(ctx->bn[ctx->tos++]);\n\tif (!BN_MONT_CTX_set(mont,a,ctx2)) goto err;\n\tfor (i=0; i<checks; i++)\n\t\t{\n\t\tif (!BN_rand(check,BN_num_bits(a)-1,0,0)) goto err;\n\t\tj=witness(check,a,ctx,ctx2,mont);\n\t\tif (j == -1) goto err;\n\t\tif (j)\n\t\t\t{\n\t\t\tret=0;\n\t\t\tgoto err;\n\t\t\t}\n\t\tif (callback != NULL) callback(1,c2++,cb_arg);\n\t\t}\n\tret=1;\nerr:\n\tctx->tos--;\n\tif ((ctx_passed == NULL) && (ctx != NULL))\n\t\tBN_CTX_free(ctx);\n\tif (ctx2 != NULL)\n\t\tBN_CTX_free(ctx2);\n\tif (mont != NULL) BN_MONT_CTX_free(mont);\n\treturn(ret);\n\t}', 'int BN_mod_mul(BIGNUM *ret, BIGNUM *a, BIGNUM *b, const BIGNUM *m, BN_CTX *ctx)\n\t{\n\tBIGNUM *t;\n\tint r=0;\n\tbn_check_top(a);\n\tbn_check_top(b);\n\tbn_check_top(m);\n\tt= &(ctx->bn[ctx->tos++]);\n\tif (a == b)\n\t\t{ if (!BN_sqr(t,a,ctx)) goto err; }\n\telse\n\t\t{ if (!BN_mul(t,a,b,ctx)) goto err; }\n\tif (!BN_mod(ret,t,m,ctx)) goto err;\n\tr=1;\nerr:\n\tctx->tos--;\n\treturn(r);\n\t}', 'int BN_sqr(BIGNUM *r, BIGNUM *a, BN_CTX *ctx)\n\t{\n\tint max,al;\n\tBIGNUM *tmp,*rr;\n#ifdef BN_COUNT\nprintf("BN_sqr %d * %d\\n",a->top,a->top);\n#endif\n\tbn_check_top(a);\n\ttmp= &(ctx->bn[ctx->tos]);\n\trr=(a != r)?r: (&ctx->bn[ctx->tos+1]);\n\tal=a->top;\n\tif (al <= 0)\n\t\t{\n\t\tr->top=0;\n\t\treturn(1);\n\t\t}\n\tmax=(al+al);\n\tif (bn_wexpand(rr,max+1) == NULL) return(0);\n\tr->neg=0;\n\tif (al == 4)\n\t\t{\n#ifndef BN_SQR_COMBA\n\t\tBN_ULONG t[8];\n\t\tbn_sqr_normal(rr->d,a->d,4,t);\n#else\n\t\tbn_sqr_comba4(rr->d,a->d);\n#endif\n\t\t}\n\telse if (al == 8)\n\t\t{\n#ifndef BN_SQR_COMBA\n\t\tBN_ULONG t[16];\n\t\tbn_sqr_normal(rr->d,a->d,8,t);\n#else\n\t\tbn_sqr_comba8(rr->d,a->d);\n#endif\n\t\t}\n\telse\n\t\t{\n#if defined(BN_RECURSION)\n\t\tif (al < BN_SQR_RECURSIVE_SIZE_NORMAL)\n\t\t\t{\n\t\t\tBN_ULONG t[BN_SQR_RECURSIVE_SIZE_NORMAL*2];\n\t\t\tbn_sqr_normal(rr->d,a->d,al,t);\n\t\t\t}\n\t\telse\n\t\t\t{\n\t\t\tint j,k;\n\t\t\tj=BN_num_bits_word((BN_ULONG)al);\n\t\t\tj=1<<(j-1);\n\t\t\tk=j+j;\n\t\t\tif (al == j)\n\t\t\t\t{\n\t\t\t\tif (bn_wexpand(a,k*2) == NULL) return(0);\n\t\t\t\tif (bn_wexpand(tmp,k*2) == NULL) return(0);\n\t\t\t\tbn_sqr_recursive(rr->d,a->d,al,tmp->d);\n\t\t\t\t}\n\t\t\telse\n\t\t\t\t{\n\t\t\t\tif (bn_wexpand(tmp,max) == NULL) return(0);\n\t\t\t\tbn_sqr_normal(rr->d,a->d,al,tmp->d);\n\t\t\t\t}\n\t\t\t}\n#else\n\t\tif (bn_wexpand(tmp,max) == NULL) return(0);\n\t\tbn_sqr_normal(rr->d,a->d,al,tmp->d);\n#endif\n\t\t}\n\trr->top=max;\n\tif ((max > 0) && (rr->d[max-1] == 0)) rr->top--;\n\tif (rr != r) BN_copy(r,rr);\n\treturn(1);\n\t}']