crypto/rsa/rsa_oaep.c:112: error: BUFFER_OVERRUN_L2
  Offset: [20, -20+max(40, `flen`)] Size: [-20+max(41, `flen`), `flen` - 20].
Showing all 8 steps of the trace


crypto/rsa/rsa_oaep.c:100:10: <Offset trace>
98.     
99.       MGF1(db, dblen, seed, SHA_DIGEST_LENGTH);
100.       for (i = 0; i < dblen; i++)
                ^
101.   	db[i] ^= maskeddb[i];
102.   

crypto/rsa/rsa_oaep.c:100:10: Assignment
98.     
99.       MGF1(db, dblen, seed, SHA_DIGEST_LENGTH);
100.       for (i = 0; i < dblen; i++)
                ^
101.   	db[i] ^= maskeddb[i];
102.   

crypto/rsa/rsa_oaep.c:71:1: <Length trace>
69.       }
70.   
71. > int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
72.   	     unsigned char *from, int flen, int num, unsigned char *param,
73.   	     int plen)

crypto/rsa/rsa_oaep.c:71:1: Parameter `flen`
69.       }
70.   
71. > int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
72.   	     unsigned char *from, int flen, int num, unsigned char *param,
73.   	     int plen)

crypto/rsa/rsa_oaep.c:85:5: Assignment
83.   	}
84.   
85.       dblen = flen - SHA_DIGEST_LENGTH;
          ^
86.       db = Malloc(dblen);
87.       if (db == NULL)

crypto/rsa/rsa_oaep.c:86:10: Array declaration
84.   
85.       dblen = flen - SHA_DIGEST_LENGTH;
86.       db = Malloc(dblen);
               ^
87.       if (db == NULL)
88.   	{

crypto/rsa/rsa_oaep.c:86:5: Assignment
84.   
85.       dblen = flen - SHA_DIGEST_LENGTH;
86.       db = Malloc(dblen);
          ^
87.       if (db == NULL)
88.   	{

crypto/rsa/rsa_oaep.c:112:6: Array access: Offset: [20, -20+max(40, flen)] Size: [-20+max(41, flen), flen - 20]
110.   	    if (db[i] != 0x00)
111.   		break;
112.   	if (db[i] != 0x01 || i++ >= dblen)
            ^
113.   	    RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP,
114.   		   RSA_R_OAEP_DECODING_ERROR);

int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
	     unsigned char *from, int flen, int num, unsigned char *param,
	     int plen)
    {
    int i, dblen, mlen = -1;
    unsigned char *maskeddb;
    unsigned char *db, seed[SHA_DIGEST_LENGTH], phash[SHA_DIGEST_LENGTH];
    if (flen < 2 * SHA_DIGEST_LENGTH + 1)
	{
	RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, RSA_R_OAEP_DECODING_ERROR);
	return (-1);
	}
    dblen = flen - SHA_DIGEST_LENGTH;
    db = Malloc(dblen);
    if (db == NULL)
	{
	RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);
	return (-1);
	}
    maskeddb = from + SHA_DIGEST_LENGTH;
    MGF1(seed, SHA_DIGEST_LENGTH, maskeddb, dblen);
    for (i = 0; i < SHA_DIGEST_LENGTH; i++)
	seed[i] ^= from[i];
    MGF1(db, dblen, seed, SHA_DIGEST_LENGTH);
    for (i = 0; i < dblen; i++)
	db[i] ^= maskeddb[i];
    SHA1(param, plen, phash);
    if (memcmp(db, phash, SHA_DIGEST_LENGTH) != 0)
	RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, RSA_R_OAEP_DECODING_ERROR);
    else
	{
	for (i = SHA_DIGEST_LENGTH; i < dblen; i++)
	    if (db[i] != 0x00)
		break;
	if (db[i] != 0x01 || i++ >= dblen)
	    RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP,
		   RSA_R_OAEP_DECODING_ERROR);
	else
	    {
	    mlen = dblen - i;
	    if (tlen < mlen)
		{
		RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, RSA_R_DATA_TOO_LARGE);
		mlen = -1;
		}
	    else
		memcpy(to, db + i, mlen);
	    }
	}
    Free(db);
    return (mlen);
    }
['int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,\n\t     unsigned char *from, int flen, int num, unsigned char *param,\n\t     int plen)\n    {\n    int i, dblen, mlen = -1;\n    unsigned char *maskeddb;\n    unsigned char *db, seed[SHA_DIGEST_LENGTH], phash[SHA_DIGEST_LENGTH];\n    if (flen < 2 * SHA_DIGEST_LENGTH + 1)\n\t{\n\tRSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, RSA_R_OAEP_DECODING_ERROR);\n\treturn (-1);\n\t}\n    dblen = flen - SHA_DIGEST_LENGTH;\n    db = Malloc(dblen);\n    if (db == NULL)\n\t{\n\tRSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);\n\treturn (-1);\n\t}\n    maskeddb = from + SHA_DIGEST_LENGTH;\n    MGF1(seed, SHA_DIGEST_LENGTH, maskeddb, dblen);\n    for (i = 0; i < SHA_DIGEST_LENGTH; i++)\n\tseed[i] ^= from[i];\n    MGF1(db, dblen, seed, SHA_DIGEST_LENGTH);\n    for (i = 0; i < dblen; i++)\n\tdb[i] ^= maskeddb[i];\n    SHA1(param, plen, phash);\n    if (memcmp(db, phash, SHA_DIGEST_LENGTH) != 0)\n\tRSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, RSA_R_OAEP_DECODING_ERROR);\n    else\n\t{\n\tfor (i = SHA_DIGEST_LENGTH; i < dblen; i++)\n\t    if (db[i] != 0x00)\n\t\tbreak;\n\tif (db[i] != 0x01 || i++ >= dblen)\n\t    RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP,\n\t\t   RSA_R_OAEP_DECODING_ERROR);\n\telse\n\t    {\n\t    mlen = dblen - i;\n\t    if (tlen < mlen)\n\t\t{\n\t\tRSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, RSA_R_DATA_TOO_LARGE);\n\t\tmlen = -1;\n\t\t}\n\t    else\n\t\tmemcpy(to, db + i, mlen);\n\t    }\n\t}\n    Free(db);\n    return (mlen);\n    }']