crypto/rsa/rsa_eay.c:120: error: BUFFER_OVERRUN_L3
  Offset: [5, +oo] (⇐ [2, +oo] + 3) Size: [0, +oo] by call to `RSA_padding_add_PKCS1_OAEP`.
Showing all 14 steps of the trace


crypto/rsa/rsa_eay.c:106:6: Call
104.   	BN_init(&ret);
105.   	if ((ctx=BN_CTX_new()) == NULL) goto err;
106.   	num=BN_num_bytes(rsa->n);
            ^
107.   	if ((buf=(unsigned char *)Malloc(num)) == NULL)
108.   		{

crypto/bn/bn_lib.c:245:19: Assignment
  243.   	bn_check_top(a);
  244.   
  245.   	if (a->top == 0) return(0);
                           ^
  246.   	l=a->d[a->top-1];
  247.   	i=(a->top-1)*BN_BITS2;

crypto/rsa/rsa_eay.c:106:2: Assignment
104.   	BN_init(&ret);
105.   	if ((ctx=BN_CTX_new()) == NULL) goto err;
106.   	num=BN_num_bytes(rsa->n);
        ^
107.   	if ((buf=(unsigned char *)Malloc(num)) == NULL)
108.   		{

crypto/rsa/rsa_eay.c:107:28: Array declaration
105.   	if ((ctx=BN_CTX_new()) == NULL) goto err;
106.   	num=BN_num_bytes(rsa->n);
107.   	if ((buf=(unsigned char *)Malloc(num)) == NULL)
                                  ^
108.   		{
109.   		RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,ERR_R_MALLOC_FAILURE);

crypto/rsa/rsa_eay.c:107:7: Assignment
105.   	if ((ctx=BN_CTX_new()) == NULL) goto err;
106.   	num=BN_num_bytes(rsa->n);
107.   	if ((buf=(unsigned char *)Malloc(num)) == NULL)
             ^
108.   		{
109.   		RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,ERR_R_MALLOC_FAILURE);

crypto/rsa/rsa_eay.c:120:12: Call
118.   #ifndef NO_SHA
119.   	case RSA_PKCS1_OAEP_PADDING:
120.   	        i=RSA_padding_add_PKCS1_OAEP(buf,num,from,flen,NULL,0);
                  ^
121.   		break;
122.   #endif

crypto/rsa/rsa_oaep.c:17:1: Parameter `*to`
  15.   int MGF1(unsigned char *mask, long len, unsigned char *seed, long seedlen);
  16.   
  17. > int RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen,
  18.   	     unsigned char *from, int flen, unsigned char *param, int plen)
  19.       {

crypto/rsa/rsa_oaep.c:45:5: Assignment
  43.   
  44.       to[0] = 0;
  45.       seed = to + 1;
            ^
  46.       db = to + SHA_DIGEST_LENGTH + 1;
  47.   

crypto/rsa/rsa_oaep.c:60:5: Call
  58.   #endif
  59.   
  60.       MGF1(dbmask, emlen - SHA_DIGEST_LENGTH, seed, SHA_DIGEST_LENGTH);
            ^
  61.       for (i = 0; i < emlen - SHA_DIGEST_LENGTH; i++)
  62.   	db[i] ^= dbmask[i];

crypto/rsa/rsa_oaep.c:134:1: Parameter `*seed`
    132.       }
    133.   
    134. > int MGF1(unsigned char *mask, long len, unsigned char *seed, long seedlen)
    135.       {
    136.       long i, outlen = 0;

crypto/rsa/rsa_oaep.c:146:2: Call
    144.   	  cnt[2] = (i >> 8) & 255, cnt[3] = i & 255;
    145.   	SHA1_Init(&c);
    146.   	SHA1_Update(&c, seed, seedlen);
            ^
    147.   	SHA1_Update(&c, cnt, 4);
    148.   	if (outlen + SHA_DIGEST_LENGTH <= len)

crypto/sha/sha1dgst.c:117:1: <Length trace>
115.   	}
116.   
117. > void SHA1_Update(SHA_CTX *c, const register unsigned char *data,
118.   	     unsigned long len)
119.   	{

crypto/sha/sha1dgst.c:117:1: Parameter `*data`
      115.   	}
      116.   
      117. > void SHA1_Update(SHA_CTX *c, const register unsigned char *data,
      118.   	     unsigned long len)
      119.   	{

crypto/sha/sha1dgst.c:232:5: Array access: Offset: [5, +oo] (⇐ [2, +oo] + 3) Size: [0, +oo] by call to `RSA_padding_add_PKCS1_OAEP` 
      230.   				BE_COPY(p,data,1);
      231.   				BE_COPY(p,data,2);
      232.   				BE_COPY(p,data,3);
                 ^
      233.   				p+=4;
      234.   				data += 4*sizeof(SHA_LONG);

void SHA1_Update(SHA_CTX *c, const register unsigned char *data,
	     unsigned long len)
	{
	register SHA_LONG *p;
	int ew,ec,sw,sc;
	SHA_LONG l;
	if (len == 0) return;
	l=(c->Nl+(len<<3))&0xffffffffL;
	if (l < c->Nl)
		c->Nh++;
	c->Nh+=(len>>29);
	c->Nl=l;
	if (c->num != 0)
		{
		p=c->data;
		sw=c->num>>2;
		sc=c->num&0x03;
		if ((c->num+len) >= SHA_CBLOCK)
			{
			l= p[sw];
			M_p_c2nl(data,l,sc);
			p[sw++]=l;
			for (; sw<SHA_LBLOCK; sw++)
				{
				M_c2nl(data,l);
				p[sw]=l;
				}
			len-=(SHA_CBLOCK-c->num);
			sha1_block(c,p,1);
			c->num=0;
			}
		else
			{
			c->num+=(int)len;
			if ((sc+len) < 4)
				{
				l= p[sw];
				M_p_c2nl_p(data,l,sc,len);
				p[sw]=l;
				}
			else
				{
				ew=(c->num>>2);
				ec=(c->num&0x03);
				l= p[sw];
				M_p_c2nl(data,l,sc);
				p[sw++]=l;
				for (; sw < ew; sw++)
					{ M_c2nl(data,l); p[sw]=l; }
				if (ec)
					{
					M_c2nl_p(data,l,ec);
					p[sw]=l;
					}
				}
			return;
			}
		}
#if SHA_LONG_LOG2==2
#if defined(B_ENDIAN) || defined(SHA1_ASM)
	if ((((unsigned long)data)%sizeof(SHA_LONG)) == 0)
		{
		sw=len/SHA_CBLOCK;
		if (sw)
			{
			sha1_block(c,(SHA_LONG *)data,sw);
			sw*=SHA_CBLOCK;
			data+=sw;
			len-=sw;
			}
		}
#endif
#endif
	p=c->data;
	while (len >= SHA_CBLOCK)
		{
#if SHA_LONG_LOG2==2
#if defined(B_ENDIAN) || defined(SHA1_ASM)
#define SHA_NO_TAIL_CODE
		if (p != (SHA_LONG *)data)
			memcpy(p,data,SHA_CBLOCK);
		data+=SHA_CBLOCK;
		sha1_block(c,p=c->data,1);
		len-=SHA_CBLOCK;
#else
#define BE_COPY(dst,src,i)	{				\
				l = ((SHA_LONG *)src)[i];	\
				Endian_Reverse32(l);		\
				dst[i] = l;			\
				}
		if ((((unsigned long)data)%sizeof(SHA_LONG)) == 0)
			{
			for (sw=(SHA_LBLOCK/4); sw; sw--)
				{
				BE_COPY(p,data,0);
				BE_COPY(p,data,1);
				BE_COPY(p,data,2);
				BE_COPY(p,data,3);
				p+=4;
				data += 4*sizeof(SHA_LONG);
				}
			sha1_block(c,p=c->data,1);
			len-=SHA_CBLOCK;
			continue;
			}
#endif
#endif
#ifndef SHA_NO_TAIL_CODE
		p=c->data;
		for (sw=(SHA_LBLOCK/4); sw; sw--)
			{
			M_c2nl(data,l); p[0]=l;
			M_c2nl(data,l); p[1]=l;
			M_c2nl(data,l); p[2]=l;
			M_c2nl(data,l); p[3]=l;
			p+=4;
			}
		p=c->data;
		sha1_block(c,p,1);
		len-=SHA_CBLOCK;
#endif
		}
	ec=(int)len;
	c->num=ec;
	ew=(ec>>2);
	ec&=0x03;
	for (sw=0; sw < ew; sw++)
		{ M_c2nl(data,l); p[sw]=l; }
	M_c2nl_p(data,l,ec);
	p[sw]=l;
	}
['static int RSA_eay_public_encrypt(int flen, unsigned char *from,\n\t     unsigned char *to, RSA *rsa, int padding)\n\t{\n\tBIGNUM f,ret;\n\tint i,j,k,num=0,r= -1;\n\tunsigned char *buf=NULL;\n\tBN_CTX *ctx=NULL;\n\tBN_init(&f);\n\tBN_init(&ret);\n\tif ((ctx=BN_CTX_new()) == NULL) goto err;\n\tnum=BN_num_bytes(rsa->n);\n\tif ((buf=(unsigned char *)Malloc(num)) == NULL)\n\t\t{\n\t\tRSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,ERR_R_MALLOC_FAILURE);\n\t\tgoto err;\n\t\t}\n\tswitch (padding)\n\t\t{\n\tcase RSA_PKCS1_PADDING:\n\t\ti=RSA_padding_add_PKCS1_type_2(buf,num,from,flen);\n\t\tbreak;\n#ifndef NO_SHA\n\tcase RSA_PKCS1_OAEP_PADDING:\n\t        i=RSA_padding_add_PKCS1_OAEP(buf,num,from,flen,NULL,0);\n\t\tbreak;\n#endif\n\tcase RSA_SSLV23_PADDING:\n\t\ti=RSA_padding_add_SSLv23(buf,num,from,flen);\n\t\tbreak;\n\tcase RSA_NO_PADDING:\n\t\ti=RSA_padding_add_none(buf,num,from,flen);\n\t\tbreak;\n\tdefault:\n\t\tRSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,RSA_R_UNKNOWN_PADDING_TYPE);\n\t\tgoto err;\n\t\t}\n\tif (i <= 0) goto err;\n\tif (BN_bin2bn(buf,num,&f) == NULL) goto err;\n\tif ((rsa->_method_mod_n == NULL) && (rsa->flags & RSA_FLAG_CACHE_PUBLIC))\n\t\t{\n\t\tif ((rsa->_method_mod_n=BN_MONT_CTX_new()) != NULL)\n\t\t\tif (!BN_MONT_CTX_set(rsa->_method_mod_n,rsa->n,ctx))\n\t\t\t    goto err;\n\t\t}\n\tif (!rsa->meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx,\n\t\trsa->_method_mod_n)) goto err;\n\tj=BN_num_bytes(&ret);\n\ti=BN_bn2bin(&ret,&(to[num-j]));\n\tfor (k=0; k<(num-i); k++)\n\t\tto[k]=0;\n\tr=num;\nerr:\n\tif (ctx != NULL) BN_CTX_free(ctx);\n\tBN_clear_free(&f);\n\tBN_clear_free(&ret);\n\tif (buf != NULL)\n\t\t{\n\t\tmemset(buf,0,num);\n\t\tFree(buf);\n\t\t}\n\treturn(r);\n\t}', 'int BN_num_bits(BIGNUM *a)\n\t{\n\tBN_ULONG l;\n\tint i;\n\tbn_check_top(a);\n\tif (a->top == 0) return(0);\n\tl=a->d[a->top-1];\n\ti=(a->top-1)*BN_BITS2;\n\tif (l == 0)\n\t\t{\n#if !defined(NO_STDIO) && !defined(WIN16)\n\t\tfprintf(stderr,"BAD TOP VALUE\\n");\n#endif\n\t\tabort();\n\t\t}\n\treturn(i+BN_num_bits_word(l));\n\t}', 'int RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen,\n\t     unsigned char *from, int flen, unsigned char *param, int plen)\n    {\n    int i, emlen = tlen - 1;\n    unsigned char *db, *seed;\n    unsigned char *dbmask, seedmask[SHA_DIGEST_LENGTH];\n    if (flen > emlen - 2 * SHA_DIGEST_LENGTH - 1)\n\t{\n\tRSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP,\n\t       RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);\n\treturn (0);\n\t}\n    if (emlen < 2 * SHA_DIGEST_LENGTH + 1)\n\t{\n\tRSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, RSA_R_KEY_SIZE_TOO_SMALL);\n\treturn (0);\n\t}\n    dbmask = Malloc(emlen - SHA_DIGEST_LENGTH);\n    if (dbmask == NULL)\n\t{\n\tRSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);\n\treturn (0);\n\t}\n    to[0] = 0;\n    seed = to + 1;\n    db = to + SHA_DIGEST_LENGTH + 1;\n    SHA1(param, plen, db);\n    memset(db + SHA_DIGEST_LENGTH, 0,\n\t   emlen - flen - 2 * SHA_DIGEST_LENGTH - 1);\n    db[emlen - flen - SHA_DIGEST_LENGTH - 1] = 0x01;\n    memcpy(db + emlen - flen - SHA_DIGEST_LENGTH, from, (unsigned int) flen);\n    RAND_bytes(seed, SHA_DIGEST_LENGTH);\n#ifdef PKCS_TESTVECT\n    memcpy(seed,\n\t   "\\xaa\\xfd\\x12\\xf6\\x59\\xca\\xe6\\x34\\x89\\xb4\\x79\\xe5\\x07\\x6d\\xde\\xc2\\xf0\\x6c\\xb5\\x8f",\n\t   20);\n#endif\n    MGF1(dbmask, emlen - SHA_DIGEST_LENGTH, seed, SHA_DIGEST_LENGTH);\n    for (i = 0; i < emlen - SHA_DIGEST_LENGTH; i++)\n\tdb[i] ^= dbmask[i];\n    MGF1(seedmask, SHA_DIGEST_LENGTH, db, emlen - SHA_DIGEST_LENGTH);\n    for (i = 0; i < SHA_DIGEST_LENGTH; i++)\n\tseed[i] ^= seedmask[i];\n    Free(dbmask);\n    return (1);\n    }', 'int MGF1(unsigned char *mask, long len, unsigned char *seed, long seedlen)\n    {\n    long i, outlen = 0;\n    unsigned char cnt[4];\n    SHA_CTX c;\n    unsigned char md[SHA_DIGEST_LENGTH];\n    for (i = 0; outlen < len; i++)\n\t{\n\tcnt[0] = (i >> 24) & 255, cnt[1] = (i >> 16) & 255,\n\t  cnt[2] = (i >> 8) & 255, cnt[3] = i & 255;\n\tSHA1_Init(&c);\n\tSHA1_Update(&c, seed, seedlen);\n\tSHA1_Update(&c, cnt, 4);\n\tif (outlen + SHA_DIGEST_LENGTH <= len)\n\t    {\n\t    SHA1_Final(mask + outlen, &c);\n\t    outlen += SHA_DIGEST_LENGTH;\n\t    }\n\telse\n\t    {\n\t    SHA1_Final(md, &c);\n\t    memcpy(mask + outlen, md, len - outlen);\n\t    outlen = len;\n\t    }\n\t}\n    return (0);\n    }', 'void SHA1_Update(SHA_CTX *c, const register unsigned char *data,\n\t     unsigned long len)\n\t{\n\tregister SHA_LONG *p;\n\tint ew,ec,sw,sc;\n\tSHA_LONG l;\n\tif (len == 0) return;\n\tl=(c->Nl+(len<<3))&0xffffffffL;\n\tif (l < c->Nl)\n\t\tc->Nh++;\n\tc->Nh+=(len>>29);\n\tc->Nl=l;\n\tif (c->num != 0)\n\t\t{\n\t\tp=c->data;\n\t\tsw=c->num>>2;\n\t\tsc=c->num&0x03;\n\t\tif ((c->num+len) >= SHA_CBLOCK)\n\t\t\t{\n\t\t\tl= p[sw];\n\t\t\tM_p_c2nl(data,l,sc);\n\t\t\tp[sw++]=l;\n\t\t\tfor (; sw<SHA_LBLOCK; sw++)\n\t\t\t\t{\n\t\t\t\tM_c2nl(data,l);\n\t\t\t\tp[sw]=l;\n\t\t\t\t}\n\t\t\tlen-=(SHA_CBLOCK-c->num);\n\t\t\tsha1_block(c,p,1);\n\t\t\tc->num=0;\n\t\t\t}\n\t\telse\n\t\t\t{\n\t\t\tc->num+=(int)len;\n\t\t\tif ((sc+len) < 4)\n\t\t\t\t{\n\t\t\t\tl= p[sw];\n\t\t\t\tM_p_c2nl_p(data,l,sc,len);\n\t\t\t\tp[sw]=l;\n\t\t\t\t}\n\t\t\telse\n\t\t\t\t{\n\t\t\t\tew=(c->num>>2);\n\t\t\t\tec=(c->num&0x03);\n\t\t\t\tl= p[sw];\n\t\t\t\tM_p_c2nl(data,l,sc);\n\t\t\t\tp[sw++]=l;\n\t\t\t\tfor (; sw < ew; sw++)\n\t\t\t\t\t{ M_c2nl(data,l); p[sw]=l; }\n\t\t\t\tif (ec)\n\t\t\t\t\t{\n\t\t\t\t\tM_c2nl_p(data,l,ec);\n\t\t\t\t\tp[sw]=l;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\treturn;\n\t\t\t}\n\t\t}\n#if SHA_LONG_LOG2==2\n#if defined(B_ENDIAN) || defined(SHA1_ASM)\n\tif ((((unsigned long)data)%sizeof(SHA_LONG)) == 0)\n\t\t{\n\t\tsw=len/SHA_CBLOCK;\n\t\tif (sw)\n\t\t\t{\n\t\t\tsha1_block(c,(SHA_LONG *)data,sw);\n\t\t\tsw*=SHA_CBLOCK;\n\t\t\tdata+=sw;\n\t\t\tlen-=sw;\n\t\t\t}\n\t\t}\n#endif\n#endif\n\tp=c->data;\n\twhile (len >= SHA_CBLOCK)\n\t\t{\n#if SHA_LONG_LOG2==2\n#if defined(B_ENDIAN) || defined(SHA1_ASM)\n#define SHA_NO_TAIL_CODE\n\t\tif (p != (SHA_LONG *)data)\n\t\t\tmemcpy(p,data,SHA_CBLOCK);\n\t\tdata+=SHA_CBLOCK;\n\t\tsha1_block(c,p=c->data,1);\n\t\tlen-=SHA_CBLOCK;\n#else\n#define BE_COPY(dst,src,i)\t{\t\t\t\t\\\n\t\t\t\tl = ((SHA_LONG *)src)[i];\t\\\n\t\t\t\tEndian_Reverse32(l);\t\t\\\n\t\t\t\tdst[i] = l;\t\t\t\\\n\t\t\t\t}\n\t\tif ((((unsigned long)data)%sizeof(SHA_LONG)) == 0)\n\t\t\t{\n\t\t\tfor (sw=(SHA_LBLOCK/4); sw; sw--)\n\t\t\t\t{\n\t\t\t\tBE_COPY(p,data,0);\n\t\t\t\tBE_COPY(p,data,1);\n\t\t\t\tBE_COPY(p,data,2);\n\t\t\t\tBE_COPY(p,data,3);\n\t\t\t\tp+=4;\n\t\t\t\tdata += 4*sizeof(SHA_LONG);\n\t\t\t\t}\n\t\t\tsha1_block(c,p=c->data,1);\n\t\t\tlen-=SHA_CBLOCK;\n\t\t\tcontinue;\n\t\t\t}\n#endif\n#endif\n#ifndef SHA_NO_TAIL_CODE\n\t\tp=c->data;\n\t\tfor (sw=(SHA_LBLOCK/4); sw; sw--)\n\t\t\t{\n\t\t\tM_c2nl(data,l); p[0]=l;\n\t\t\tM_c2nl(data,l); p[1]=l;\n\t\t\tM_c2nl(data,l); p[2]=l;\n\t\t\tM_c2nl(data,l); p[3]=l;\n\t\t\tp+=4;\n\t\t\t}\n\t\tp=c->data;\n\t\tsha1_block(c,p,1);\n\t\tlen-=SHA_CBLOCK;\n#endif\n\t\t}\n\tec=(int)len;\n\tc->num=ec;\n\tew=(ec>>2);\n\tec&=0x03;\n\tfor (sw=0; sw < ew; sw++)\n\t\t{ M_c2nl(data,l); p[sw]=l; }\n\tM_c2nl_p(data,l,ec);\n\tp[sw]=l;\n\t}']