crypto/rsa/rsa_eay.c:306: error: BUFFER_OVERRUN_L3
  Offset: [20, +oo] (⇐ 20 + [0, +oo]) Size: [0, +oo] by call to `RSA_padding_check_PKCS1_OAEP`.
Showing all 12 steps of the trace


crypto/rsa/rsa_eay.c:256:6: Call
254.   	if (ctx == NULL) goto err;
255.   
256.   	num=BN_num_bytes(rsa->n);
            ^
257.   
258.   	if ((buf=(unsigned char *)Malloc(num)) == NULL)

crypto/bn/bn_lib.c:245:19: Assignment
  243.   	bn_check_top(a);
  244.   
  245.   	if (a->top == 0) return(0);
                           ^
  246.   	l=a->d[a->top-1];
  247.   	i=(a->top-1)*BN_BITS2;

crypto/rsa/rsa_eay.c:256:2: Assignment
254.   	if (ctx == NULL) goto err;
255.   
256.   	num=BN_num_bytes(rsa->n);
        ^
257.   
258.   	if ((buf=(unsigned char *)Malloc(num)) == NULL)

crypto/rsa/rsa_eay.c:258:28: Array declaration
256.   	num=BN_num_bytes(rsa->n);
257.   
258.   	if ((buf=(unsigned char *)Malloc(num)) == NULL)
                                  ^
259.   		{
260.   		RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,ERR_R_MALLOC_FAILURE);

crypto/rsa/rsa_eay.c:258:7: Assignment
256.   	num=BN_num_bytes(rsa->n);
257.   
258.   	if ((buf=(unsigned char *)Malloc(num)) == NULL)
             ^
259.   		{
260.   		RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,ERR_R_MALLOC_FAILURE);

crypto/rsa/rsa_eay.c:306:12: Call
304.   #ifndef NO_SHA
305.           case RSA_PKCS1_OAEP_PADDING:
306.   	        r=RSA_padding_check_PKCS1_OAEP(to,num,buf,j,num,NULL,0);
                  ^
307.                   break;
308.   #endif

crypto/rsa/rsa_oaep.c:101:10: <Offset trace>
99.     
100.       MGF1(db, dblen, seed, SHA_DIGEST_LENGTH);
101.       for (i = 0; i < dblen; i++)
                ^
102.   	db[i] ^= maskeddb[i];
103.   

crypto/rsa/rsa_oaep.c:101:10: Assignment
  99.     
  100.       MGF1(db, dblen, seed, SHA_DIGEST_LENGTH);
  101.       for (i = 0; i < dblen; i++)
                  ^
  102.   	db[i] ^= maskeddb[i];
  103.   

crypto/rsa/rsa_oaep.c:72:1: <Length trace>
70.       }
71.   
72. > int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
73.   	     unsigned char *from, int flen, int num, unsigned char *param,
74.   	     int plen)

crypto/rsa/rsa_oaep.c:72:1: Parameter `*from`
  70.       }
  71.   
  72. > int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
  73.   	     unsigned char *from, int flen, int num, unsigned char *param,
  74.   	     int plen)

crypto/rsa/rsa_oaep.c:94:5: Assignment
  92.   	}
  93.   
  94.       maskeddb = from + SHA_DIGEST_LENGTH;
            ^
  95.       
  96.       MGF1(seed, SHA_DIGEST_LENGTH, maskeddb, dblen);

crypto/rsa/rsa_oaep.c:102:11: Array access: Offset: [20, +oo] (⇐ 20 + [0, +oo]) Size: [0, +oo] by call to `RSA_padding_check_PKCS1_OAEP` 
  100.       MGF1(db, dblen, seed, SHA_DIGEST_LENGTH);
  101.       for (i = 0; i < dblen; i++)
  102.   	db[i] ^= maskeddb[i];
                   ^
  103.   
  104.       SHA1(param, plen, phash);

int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
	     unsigned char *from, int flen, int num, unsigned char *param,
	     int plen)
    {
    int i, dblen, mlen = -1;
    unsigned char *maskeddb;
    unsigned char *db, seed[SHA_DIGEST_LENGTH], phash[SHA_DIGEST_LENGTH];
    if (flen < 2 * SHA_DIGEST_LENGTH + 1)
	{
	RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, RSA_R_OAEP_DECODING_ERROR);
	return (-1);
	}
    dblen = flen - SHA_DIGEST_LENGTH;
    db = Malloc(dblen);
    if (db == NULL)
	{
	RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);
	return (-1);
	}
    maskeddb = from + SHA_DIGEST_LENGTH;
    MGF1(seed, SHA_DIGEST_LENGTH, maskeddb, dblen);
    for (i = 0; i < SHA_DIGEST_LENGTH; i++)
	seed[i] ^= from[i];
    MGF1(db, dblen, seed, SHA_DIGEST_LENGTH);
    for (i = 0; i < dblen; i++)
	db[i] ^= maskeddb[i];
    SHA1(param, plen, phash);
    if (memcmp(db, phash, SHA_DIGEST_LENGTH) != 0)
	RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, RSA_R_OAEP_DECODING_ERROR);
    else
	{
	for (i = SHA_DIGEST_LENGTH; i < dblen; i++)
	    if (db[i] != 0x00)
		break;
	if (db[i] != 0x01 || i++ >= dblen)
	    RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP,
		   RSA_R_OAEP_DECODING_ERROR);
	else
	    {
	    mlen = dblen - i;
	    if (tlen < mlen)
		{
		RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, RSA_R_DATA_TOO_LARGE);
		mlen = -1;
		}
	    else
		memcpy(to, db + i, mlen);
	    }
	}
    Free(db);
    return (mlen);
    }
['static int RSA_eay_private_decrypt(int flen, unsigned char *from,\n\t     unsigned char *to, RSA *rsa, int padding)\n\t{\n\tBIGNUM f,ret;\n\tint j,num=0,r= -1;\n\tunsigned char *p;\n\tunsigned char *buf=NULL;\n\tBN_CTX *ctx=NULL;\n\tBN_init(&f);\n\tBN_init(&ret);\n\tctx=BN_CTX_new();\n\tif (ctx == NULL) goto err;\n\tnum=BN_num_bytes(rsa->n);\n\tif ((buf=(unsigned char *)Malloc(num)) == NULL)\n\t\t{\n\t\tRSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,ERR_R_MALLOC_FAILURE);\n\t\tgoto err;\n\t\t}\n\tif (flen > num)\n\t\t{\n\t\tRSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_DATA_GREATER_THAN_MOD_LEN);\n\t\tgoto err;\n\t\t}\n\tif (BN_bin2bn(from,(int)flen,&f) == NULL) goto err;\n\tif ((rsa->flags & RSA_FLAG_BLINDING) && (rsa->blinding == NULL))\n\t\tRSA_blinding_on(rsa,ctx);\n\tif (rsa->flags & RSA_FLAG_BLINDING)\n\t\tif (!BN_BLINDING_convert(&f,rsa->blinding,ctx)) goto err;\n\tif (\t(rsa->p != NULL) &&\n\t\t(rsa->q != NULL) &&\n\t\t(rsa->dmp1 != NULL) &&\n\t\t(rsa->dmq1 != NULL) &&\n\t\t(rsa->iqmp != NULL))\n\t\t{ if (!rsa->meth->rsa_mod_exp(&ret,&f,rsa)) goto err; }\n\telse\n\t\t{\n\t\tif (!rsa->meth->bn_mod_exp(&ret,&f,rsa->d,rsa->n,ctx,NULL))\n\t\t\tgoto err;\n\t\t}\n\tif (rsa->flags & RSA_FLAG_BLINDING)\n\t\tif (!BN_BLINDING_invert(&ret,rsa->blinding,ctx)) goto err;\n\tp=buf;\n\tj=BN_bn2bin(&ret,p);\n\tswitch (padding)\n\t\t{\n\tcase RSA_PKCS1_PADDING:\n\t\tr=RSA_padding_check_PKCS1_type_2(to,num,buf,j,num);\n\t\tbreak;\n#ifndef NO_SHA\n        case RSA_PKCS1_OAEP_PADDING:\n\t        r=RSA_padding_check_PKCS1_OAEP(to,num,buf,j,num,NULL,0);\n                break;\n#endif\n \tcase RSA_SSLV23_PADDING:\n\t\tr=RSA_padding_check_SSLv23(to,num,buf,j,num);\n\t\tbreak;\n\tcase RSA_NO_PADDING:\n\t\tr=RSA_padding_check_none(to,num,buf,j,num);\n\t\tbreak;\n\tdefault:\n\t\tRSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_UNKNOWN_PADDING_TYPE);\n\t\tgoto err;\n\t\t}\n\tif (r < 0)\n\t\tRSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_PADDING_CHECK_FAILED);\nerr:\n\tif (ctx != NULL) BN_CTX_free(ctx);\n\tBN_clear_free(&f);\n\tBN_clear_free(&ret);\n\tif (buf != NULL)\n\t\t{\n\t\tmemset(buf,0,num);\n\t\tFree(buf);\n\t\t}\n\treturn(r);\n\t}', 'int BN_num_bits(BIGNUM *a)\n\t{\n\tBN_ULONG l;\n\tint i;\n\tbn_check_top(a);\n\tif (a->top == 0) return(0);\n\tl=a->d[a->top-1];\n\ti=(a->top-1)*BN_BITS2;\n\tif (l == 0)\n\t\t{\n#if !defined(NO_STDIO) && !defined(WIN16)\n\t\tfprintf(stderr,"BAD TOP VALUE\\n");\n#endif\n\t\tabort();\n\t\t}\n\treturn(i+BN_num_bits_word(l));\n\t}', 'int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,\n\t     unsigned char *from, int flen, int num, unsigned char *param,\n\t     int plen)\n    {\n    int i, dblen, mlen = -1;\n    unsigned char *maskeddb;\n    unsigned char *db, seed[SHA_DIGEST_LENGTH], phash[SHA_DIGEST_LENGTH];\n    if (flen < 2 * SHA_DIGEST_LENGTH + 1)\n\t{\n\tRSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, RSA_R_OAEP_DECODING_ERROR);\n\treturn (-1);\n\t}\n    dblen = flen - SHA_DIGEST_LENGTH;\n    db = Malloc(dblen);\n    if (db == NULL)\n\t{\n\tRSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);\n\treturn (-1);\n\t}\n    maskeddb = from + SHA_DIGEST_LENGTH;\n    MGF1(seed, SHA_DIGEST_LENGTH, maskeddb, dblen);\n    for (i = 0; i < SHA_DIGEST_LENGTH; i++)\n\tseed[i] ^= from[i];\n    MGF1(db, dblen, seed, SHA_DIGEST_LENGTH);\n    for (i = 0; i < dblen; i++)\n\tdb[i] ^= maskeddb[i];\n    SHA1(param, plen, phash);\n    if (memcmp(db, phash, SHA_DIGEST_LENGTH) != 0)\n\tRSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, RSA_R_OAEP_DECODING_ERROR);\n    else\n\t{\n\tfor (i = SHA_DIGEST_LENGTH; i < dblen; i++)\n\t    if (db[i] != 0x00)\n\t\tbreak;\n\tif (db[i] != 0x01 || i++ >= dblen)\n\t    RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP,\n\t\t   RSA_R_OAEP_DECODING_ERROR);\n\telse\n\t    {\n\t    mlen = dblen - i;\n\t    if (tlen < mlen)\n\t\t{\n\t\tRSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, RSA_R_DATA_TOO_LARGE);\n\t\tmlen = -1;\n\t\t}\n\t    else\n\t\tmemcpy(to, db + i, mlen);\n\t    }\n\t}\n    Free(db);\n    return (mlen);\n    }']