crypto/rsa/rsa_eay.c:130: error: BUFFER_OVERRUN_L3
  Offset: [1, +oo] Size: [0, +oo] by call to `RSA_padding_add_PKCS1_OAEP`.
Showing all 14 steps of the trace


crypto/rsa/rsa_eay.c:117:6: Call
115.   	BN_init(&ret);
116.   	if ((ctx=BN_CTX_new()) == NULL) goto err;
117.   	num=BN_num_bytes(rsa->n);
            ^
118.   	if ((buf=(unsigned char *)Malloc(num)) == NULL)
119.   		{

crypto/bn/bn_lib.c:245:19: Assignment
  243.   	bn_check_top(a);
  244.   
  245.   	if (a->top == 0) return(0);
                           ^
  246.   	l=a->d[a->top-1];
  247.   	i=(a->top-1)*BN_BITS2;

crypto/rsa/rsa_eay.c:117:2: Assignment
115.   	BN_init(&ret);
116.   	if ((ctx=BN_CTX_new()) == NULL) goto err;
117.   	num=BN_num_bytes(rsa->n);
        ^
118.   	if ((buf=(unsigned char *)Malloc(num)) == NULL)
119.   		{

crypto/rsa/rsa_eay.c:118:28: Array declaration
116.   	if ((ctx=BN_CTX_new()) == NULL) goto err;
117.   	num=BN_num_bytes(rsa->n);
118.   	if ((buf=(unsigned char *)Malloc(num)) == NULL)
                                  ^
119.   		{
120.   		RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,ERR_R_MALLOC_FAILURE);

crypto/rsa/rsa_eay.c:118:7: Assignment
116.   	if ((ctx=BN_CTX_new()) == NULL) goto err;
117.   	num=BN_num_bytes(rsa->n);
118.   	if ((buf=(unsigned char *)Malloc(num)) == NULL)
             ^
119.   		{
120.   		RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,ERR_R_MALLOC_FAILURE);

crypto/rsa/rsa_eay.c:130:12: Call
128.   		break;
129.   	case RSA_PKCS1_OAEP_PADDING:
130.   	        i=RSA_padding_add_PKCS1_OAEP(buf,num,from,flen,NULL,0);
                  ^
131.   		break;
132.   	case RSA_SSLV23_PADDING:

crypto/rsa/rsa_oaep.c:16:1: Parameter `*to`
  14.   int MGF1(unsigned char *mask, long len, unsigned char *seed, long seedlen);
  15.   
  16. > int RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen,
  17.   	     unsigned char *from, int flen, unsigned char *param, int plen)
  18.       {

crypto/rsa/rsa_oaep.c:44:5: Assignment
  42.   
  43.       to[0] = 0;
  44.       seed = to + 1;
            ^
  45.       db = to + SHA_DIGEST_LENGTH + 1;
  46.   

crypto/rsa/rsa_oaep.c:59:5: Call
  57.   #endif
  58.   
  59.       MGF1(dbmask, emlen - SHA_DIGEST_LENGTH, seed, SHA_DIGEST_LENGTH);
            ^
  60.       for (i = 0; i < emlen - SHA_DIGEST_LENGTH; i++)
  61.   	db[i] ^= dbmask[i];

crypto/rsa/rsa_oaep.c:131:1: Parameter `*seed`
    129.       }
    130.   
    131. > int MGF1(unsigned char *mask, long len, unsigned char *seed, long seedlen)
    132.       {
    133.       long i, outlen = 0;

crypto/rsa/rsa_oaep.c:143:2: Call
    141.   	  cnt[2] = (i >> 8) & 255, cnt[3] = i & 255;
    142.   	SHA1_Init(&c);
    143.   	SHA1_Update(&c, seed, seedlen);
            ^
    144.   	SHA1_Update(&c, cnt, 4);
    145.   	if (outlen + SHA_DIGEST_LENGTH <= len)

crypto/sha/sha1dgst.c:126:1: <Length trace>
124.   	}
125.   
126. > void SHA1_Update(SHA_CTX *c, register unsigned char *data,
127.   	     unsigned long len)
128.   	{

crypto/sha/sha1dgst.c:126:1: Parameter `*data`
      124.   	}
      125.   
      126. > void SHA1_Update(SHA_CTX *c, register unsigned char *data,
      127.   	     unsigned long len)
      128.   	{

crypto/sha/sha1dgst.c:236:4: Array access: Offset: [1, +oo] Size: [0, +oo] by call to `RSA_padding_add_PKCS1_OAEP` 
      234.   		for (sw=(SHA_BLOCK/4); sw; sw--)
      235.   			{
      236.   			M_c2nl(data,l); *(p++)=l;
                ^
      237.   			M_c2nl(data,l); *(p++)=l;
      238.   			M_c2nl(data,l); *(p++)=l;

void SHA1_Update(SHA_CTX *c, register unsigned char *data,
	     unsigned long len)
	{
	register SHA_LONG *p;
	int ew,ec,sw,sc;
	SHA_LONG l;
	if (len == 0) return;
	l=(c->Nl+(len<<3))&0xffffffffL;
	if (l < c->Nl)
		c->Nh++;
	c->Nh+=(len>>29);
	c->Nl=l;
	if (c->num != 0)
		{
		p=c->data;
		sw=c->num>>2;
		sc=c->num&0x03;
		if ((c->num+len) >= SHA_CBLOCK)
			{
			l= p[sw];
			M_p_c2nl(data,l,sc);
			p[sw++]=l;
			for (; sw<SHA_LBLOCK; sw++)
				{
				M_c2nl(data,l);
				p[sw]=l;
				}
			len-=(SHA_CBLOCK-c->num);
			sha1_block(c,p,64);
			c->num=0;
			}
		else
			{
			c->num+=(int)len;
			if ((sc+len) < 4)
				{
				l= p[sw];
				M_p_c2nl_p(data,l,sc,len);
				p[sw]=l;
				}
			else
				{
				ew=(c->num>>2);
				ec=(c->num&0x03);
				l= p[sw];
				M_p_c2nl(data,l,sc);
				p[sw++]=l;
				for (; sw < ew; sw++)
					{ M_c2nl(data,l); p[sw]=l; }
				if (ec)
					{
					M_c2nl_p(data,l,ec);
					p[sw]=l;
					}
				}
			return;
			}
		}
#if 1
#if defined(B_ENDIAN) || defined(SHA1_ASM)
	if ((((unsigned long)data)%sizeof(SHA_LONG)) == 0)
		{
		sw=len/SHA_CBLOCK;
		if (sw)
			{
			sw*=SHA_CBLOCK;
			sha1_block(c,(SHA_LONG *)data,sw);
			data+=sw;
			len-=sw;
			}
		}
#endif
#endif
	p=c->data;
	while (len >= SHA_CBLOCK)
		{
#if defined(B_ENDIAN) || defined(L_ENDIAN)
		if (p != (SHA_LONG *)data)
			memcpy(p,data,SHA_CBLOCK);
		data+=SHA_CBLOCK;
#  ifdef L_ENDIAN
#    ifndef SHA1_ASM
		for (sw=(SHA_LBLOCK/4); sw; sw--)
			{
			Endian_Reverse32(p[0]);
			Endian_Reverse32(p[1]);
			Endian_Reverse32(p[2]);
			Endian_Reverse32(p[3]);
			p+=4;
			}
		p=c->data;
#    endif
#  endif
#else
		for (sw=(SHA_BLOCK/4); sw; sw--)
			{
			M_c2nl(data,l); *(p++)=l;
			M_c2nl(data,l); *(p++)=l;
			M_c2nl(data,l); *(p++)=l;
			M_c2nl(data,l); *(p++)=l;
			}
		p=c->data;
#endif
		sha1_block(c,p,64);
		len-=SHA_CBLOCK;
		}
	ec=(int)len;
	c->num=ec;
	ew=(ec>>2);
	ec&=0x03;
	for (sw=0; sw < ew; sw++)
		{ M_c2nl(data,l); p[sw]=l; }
	M_c2nl_p(data,l,ec);
	p[sw]=l;
	}
['static int RSA_eay_public_encrypt(int flen, unsigned char *from,\n\t     unsigned char *to, RSA *rsa, int padding)\n\t{\n\tBIGNUM f,ret;\n\tint i,j,k,num=0,r= -1;\n\tunsigned char *buf=NULL;\n\tBN_CTX *ctx=NULL;\n\tBN_init(&f);\n\tBN_init(&ret);\n\tif ((ctx=BN_CTX_new()) == NULL) goto err;\n\tnum=BN_num_bytes(rsa->n);\n\tif ((buf=(unsigned char *)Malloc(num)) == NULL)\n\t\t{\n\t\tRSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,ERR_R_MALLOC_FAILURE);\n\t\tgoto err;\n\t\t}\n\tswitch (padding)\n\t\t{\n\tcase RSA_PKCS1_PADDING:\n\t\ti=RSA_padding_add_PKCS1_type_2(buf,num,from,flen);\n\t\tbreak;\n\tcase RSA_PKCS1_OAEP_PADDING:\n\t        i=RSA_padding_add_PKCS1_OAEP(buf,num,from,flen,NULL,0);\n\t\tbreak;\n\tcase RSA_SSLV23_PADDING:\n\t\ti=RSA_padding_add_SSLv23(buf,num,from,flen);\n\t\tbreak;\n\tcase RSA_NO_PADDING:\n\t\ti=RSA_padding_add_none(buf,num,from,flen);\n\t\tbreak;\n\tdefault:\n\t\tRSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,RSA_R_UNKNOWN_PADDING_TYPE);\n\t\tgoto err;\n\t\t}\n\tif (i <= 0) goto err;\n\tif (BN_bin2bn(buf,num,&f) == NULL) goto err;\n\tif ((rsa->_method_mod_n == NULL) && (rsa->flags & RSA_FLAG_CACHE_PUBLIC))\n\t\t{\n\t\tif ((rsa->_method_mod_n=BN_MONT_CTX_new()) != NULL)\n\t\t\tif (!BN_MONT_CTX_set(rsa->_method_mod_n,rsa->n,ctx))\n\t\t\t    goto err;\n\t\t}\n\tif (!rsa->meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx,\n\t\trsa->_method_mod_n)) goto err;\n\tj=BN_num_bytes(&ret);\n\ti=BN_bn2bin(&ret,&(to[num-j]));\n\tfor (k=0; k<(num-i); k++)\n\t\tto[k]=0;\n\tr=num;\nerr:\n\tif (ctx != NULL) BN_CTX_free(ctx);\n\tBN_clear_free(&f);\n\tBN_clear_free(&ret);\n\tif (buf != NULL)\n\t\t{\n\t\tmemset(buf,0,num);\n\t\tFree(buf);\n\t\t}\n\treturn(r);\n\t}', 'int BN_num_bits(BIGNUM *a)\n\t{\n\tBN_ULONG l;\n\tint i;\n\tbn_check_top(a);\n\tif (a->top == 0) return(0);\n\tl=a->d[a->top-1];\n\ti=(a->top-1)*BN_BITS2;\n\tif (l == 0)\n\t\t{\n#if !defined(NO_STDIO) && !defined(WIN16)\n\t\tfprintf(stderr,"BAD TOP VALUE\\n");\n#endif\n\t\tabort();\n\t\t}\n\treturn(i+BN_num_bits_word(l));\n\t}', 'int RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen,\n\t     unsigned char *from, int flen, unsigned char *param, int plen)\n    {\n    int i, emlen = tlen - 1;\n    unsigned char *db, *seed;\n    unsigned char *dbmask, seedmask[SHA_DIGEST_LENGTH];\n    if (flen > emlen - 2 * SHA_DIGEST_LENGTH - 1)\n\t{\n\tRSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP,\n\t       RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);\n\treturn (0);\n\t}\n    if (emlen < 2 * SHA_DIGEST_LENGTH + 1)\n\t{\n\tRSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, RSA_R_KEY_SIZE_TOO_SMALL);\n\treturn (0);\n\t}\n    dbmask = Malloc(emlen - SHA_DIGEST_LENGTH);\n    if (dbmask == NULL)\n\t{\n\tRSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);\n\treturn (0);\n\t}\n    to[0] = 0;\n    seed = to + 1;\n    db = to + SHA_DIGEST_LENGTH + 1;\n    SHA1(param, plen, db);\n    memset(db + SHA_DIGEST_LENGTH, 0,\n\t   emlen - flen - 2 * SHA_DIGEST_LENGTH - 1);\n    db[emlen - flen - SHA_DIGEST_LENGTH - 1] = 0x01;\n    memcpy(db + emlen - flen - SHA_DIGEST_LENGTH, from, (unsigned int) flen);\n    RAND_bytes(seed, SHA_DIGEST_LENGTH);\n#ifdef PKCS_TESTVECT\n    memcpy(seed,\n\t   "\\xaa\\xfd\\x12\\xf6\\x59\\xca\\xe6\\x34\\x89\\xb4\\x79\\xe5\\x07\\x6d\\xde\\xc2\\xf0\\x6c\\xb5\\x8f",\n\t   20);\n#endif\n    MGF1(dbmask, emlen - SHA_DIGEST_LENGTH, seed, SHA_DIGEST_LENGTH);\n    for (i = 0; i < emlen - SHA_DIGEST_LENGTH; i++)\n\tdb[i] ^= dbmask[i];\n    MGF1(seedmask, SHA_DIGEST_LENGTH, db, emlen - SHA_DIGEST_LENGTH);\n    for (i = 0; i < SHA_DIGEST_LENGTH; i++)\n\tseed[i] ^= seedmask[i];\n    Free(dbmask);\n    return (1);\n    }', 'int MGF1(unsigned char *mask, long len, unsigned char *seed, long seedlen)\n    {\n    long i, outlen = 0;\n    unsigned char cnt[4];\n    SHA_CTX c;\n    unsigned char md[SHA_DIGEST_LENGTH];\n    for (i = 0; outlen < len; i++)\n\t{\n\tcnt[0] = (i >> 24) & 255, cnt[1] = (i >> 16) & 255,\n\t  cnt[2] = (i >> 8) & 255, cnt[3] = i & 255;\n\tSHA1_Init(&c);\n\tSHA1_Update(&c, seed, seedlen);\n\tSHA1_Update(&c, cnt, 4);\n\tif (outlen + SHA_DIGEST_LENGTH <= len)\n\t    {\n\t    SHA1_Final(mask + outlen, &c);\n\t    outlen += SHA_DIGEST_LENGTH;\n\t    }\n\telse\n\t    {\n\t    SHA1_Final(md, &c);\n\t    memcpy(mask + outlen, md, len - outlen);\n\t    outlen = len;\n\t    }\n\t}\n    return (0);\n    }', 'void SHA1_Update(SHA_CTX *c, register unsigned char *data,\n\t     unsigned long len)\n\t{\n\tregister SHA_LONG *p;\n\tint ew,ec,sw,sc;\n\tSHA_LONG l;\n\tif (len == 0) return;\n\tl=(c->Nl+(len<<3))&0xffffffffL;\n\tif (l < c->Nl)\n\t\tc->Nh++;\n\tc->Nh+=(len>>29);\n\tc->Nl=l;\n\tif (c->num != 0)\n\t\t{\n\t\tp=c->data;\n\t\tsw=c->num>>2;\n\t\tsc=c->num&0x03;\n\t\tif ((c->num+len) >= SHA_CBLOCK)\n\t\t\t{\n\t\t\tl= p[sw];\n\t\t\tM_p_c2nl(data,l,sc);\n\t\t\tp[sw++]=l;\n\t\t\tfor (; sw<SHA_LBLOCK; sw++)\n\t\t\t\t{\n\t\t\t\tM_c2nl(data,l);\n\t\t\t\tp[sw]=l;\n\t\t\t\t}\n\t\t\tlen-=(SHA_CBLOCK-c->num);\n\t\t\tsha1_block(c,p,64);\n\t\t\tc->num=0;\n\t\t\t}\n\t\telse\n\t\t\t{\n\t\t\tc->num+=(int)len;\n\t\t\tif ((sc+len) < 4)\n\t\t\t\t{\n\t\t\t\tl= p[sw];\n\t\t\t\tM_p_c2nl_p(data,l,sc,len);\n\t\t\t\tp[sw]=l;\n\t\t\t\t}\n\t\t\telse\n\t\t\t\t{\n\t\t\t\tew=(c->num>>2);\n\t\t\t\tec=(c->num&0x03);\n\t\t\t\tl= p[sw];\n\t\t\t\tM_p_c2nl(data,l,sc);\n\t\t\t\tp[sw++]=l;\n\t\t\t\tfor (; sw < ew; sw++)\n\t\t\t\t\t{ M_c2nl(data,l); p[sw]=l; }\n\t\t\t\tif (ec)\n\t\t\t\t\t{\n\t\t\t\t\tM_c2nl_p(data,l,ec);\n\t\t\t\t\tp[sw]=l;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\treturn;\n\t\t\t}\n\t\t}\n#if 1\n#if defined(B_ENDIAN) || defined(SHA1_ASM)\n\tif ((((unsigned long)data)%sizeof(SHA_LONG)) == 0)\n\t\t{\n\t\tsw=len/SHA_CBLOCK;\n\t\tif (sw)\n\t\t\t{\n\t\t\tsw*=SHA_CBLOCK;\n\t\t\tsha1_block(c,(SHA_LONG *)data,sw);\n\t\t\tdata+=sw;\n\t\t\tlen-=sw;\n\t\t\t}\n\t\t}\n#endif\n#endif\n\tp=c->data;\n\twhile (len >= SHA_CBLOCK)\n\t\t{\n#if defined(B_ENDIAN) || defined(L_ENDIAN)\n\t\tif (p != (SHA_LONG *)data)\n\t\t\tmemcpy(p,data,SHA_CBLOCK);\n\t\tdata+=SHA_CBLOCK;\n#  ifdef L_ENDIAN\n#    ifndef SHA1_ASM\n\t\tfor (sw=(SHA_LBLOCK/4); sw; sw--)\n\t\t\t{\n\t\t\tEndian_Reverse32(p[0]);\n\t\t\tEndian_Reverse32(p[1]);\n\t\t\tEndian_Reverse32(p[2]);\n\t\t\tEndian_Reverse32(p[3]);\n\t\t\tp+=4;\n\t\t\t}\n\t\tp=c->data;\n#    endif\n#  endif\n#else\n\t\tfor (sw=(SHA_BLOCK/4); sw; sw--)\n\t\t\t{\n\t\t\tM_c2nl(data,l); *(p++)=l;\n\t\t\tM_c2nl(data,l); *(p++)=l;\n\t\t\tM_c2nl(data,l); *(p++)=l;\n\t\t\tM_c2nl(data,l); *(p++)=l;\n\t\t\t}\n\t\tp=c->data;\n#endif\n\t\tsha1_block(c,p,64);\n\t\tlen-=SHA_CBLOCK;\n\t\t}\n\tec=(int)len;\n\tc->num=ec;\n\tew=(ec>>2);\n\tec&=0x03;\n\tfor (sw=0; sw < ew; sw++)\n\t\t{ M_c2nl(data,l); p[sw]=l; }\n\tM_c2nl_p(data,l,ec);\n\tp[sw]=l;\n\t}']