crypto/rsa/rsa_pss.c:203: error: INTEGER_OVERFLOW_L2
  ([-4-max(18446744073709551615, `Hash->md_size`), +oo] - 2):signed32.
Showing all 6 steps of the trace


crypto/rsa/rsa_pss.c:169:1: <LHS trace>
167.   	}
168.   
169. > int RSA_padding_add_PKCS1_PSS(RSA *rsa, unsigned char *EM,
170.   			const unsigned char *mHash,
171.   			const EVP_MD *Hash, int sLen)

crypto/rsa/rsa_pss.c:169:1: Parameter `Hash->md_size`
167.   	}
168.   
169. > int RSA_padding_add_PKCS1_PSS(RSA *rsa, unsigned char *EM,
170.   			const unsigned char *mHash,
171.   			const EVP_MD *Hash, int sLen)

crypto/rsa/rsa_pss.c:179:9: Call
177.   	EVP_MD_CTX ctx;
178.   
179.   	hLen = EVP_MD_size(Hash);
               ^
180.   	/*
181.   	 * Negative sLen has special meanings:

crypto/evp/evp_lib.c:260:3: Assignment
  258.   	{
  259.   	if (!md)
  260.   		return -1;
           ^
  261.   	return md->md_size;
  262.   	}

crypto/rsa/rsa_pss.c:179:2: Assignment
177.   	EVP_MD_CTX ctx;
178.   
179.   	hLen = EVP_MD_size(Hash);
        ^
180.   	/*
181.   	 * Negative sLen has special meanings:

crypto/rsa/rsa_pss.c:203:3: Binary operation: ([-4-max(18446744073709551615, Hash->md_size), +oo] - 2):signed32
201.   	if (sLen == -2)
202.   		{
203.   		sLen = emLen - hLen - 2;
         ^
204.   		}
205.   	else if (emLen < (hLen + sLen + 2))

int RSA_padding_add_PKCS1_PSS(RSA *rsa, unsigned char *EM,
			const unsigned char *mHash,
			const EVP_MD *Hash, int sLen)
	{
	int i;
	int ret = 0;
	int hLen, maskedDBLen, MSBits, emLen;
	unsigned char *H, *salt = NULL, *p;
	EVP_MD_CTX ctx;
	hLen = EVP_MD_size(Hash);
	if      (sLen == -1)	sLen = hLen;
	else if (sLen == -2)	sLen = -2;
	else if (sLen < -2)
		{
		RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_PSS, RSA_R_SLEN_CHECK_FAILED);
		goto err;
		}
	MSBits = (BN_num_bits(rsa->n) - 1) & 0x7;
	emLen = RSA_size(rsa);
	if (MSBits == 0)
		{
		*EM++ = 0;
		emLen--;
		}
	if (sLen == -2)
		{
		sLen = emLen - hLen - 2;
		}
	else if (emLen < (hLen + sLen + 2))
		{
		RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_PSS,
		   RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
		goto err;
		}
	if (sLen > 0)
		{
		salt = OPENSSL_malloc(sLen);
		if (!salt)
			{
			RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_PSS,
		   		ERR_R_MALLOC_FAILURE);
			goto err;
			}
		if (!RAND_bytes(salt, sLen))
			goto err;
		}
	maskedDBLen = emLen - hLen - 1;
	H = EM + maskedDBLen;
	EVP_MD_CTX_init(&ctx);
	EVP_DigestInit_ex(&ctx, Hash, NULL);
	EVP_DigestUpdate(&ctx, zeroes, sizeof zeroes);
	EVP_DigestUpdate(&ctx, mHash, hLen);
	if (sLen)
		EVP_DigestUpdate(&ctx, salt, sLen);
	EVP_DigestFinal(&ctx, H, NULL);
	EVP_MD_CTX_cleanup(&ctx);
	PKCS1_MGF1(EM, maskedDBLen, H, hLen, Hash);
	p = EM;
	p += emLen - sLen - hLen - 2;
	*p++ ^= 0x1;
	if (sLen > 0)
		{
		for (i = 0; i < sLen; i++)
			*p++ ^= salt[i];
		}
	if (MSBits)
		EM[0] &= 0xFF >> (8 - MSBits);
	EM[emLen - 1] = 0xbc;
	ret = 1;
	err:
	if (salt)
		OPENSSL_free(salt);
	return ret;
	}
['int RSA_padding_add_PKCS1_PSS(RSA *rsa, unsigned char *EM,\n\t\t\tconst unsigned char *mHash,\n\t\t\tconst EVP_MD *Hash, int sLen)\n\t{\n\tint i;\n\tint ret = 0;\n\tint hLen, maskedDBLen, MSBits, emLen;\n\tunsigned char *H, *salt = NULL, *p;\n\tEVP_MD_CTX ctx;\n\thLen = EVP_MD_size(Hash);\n\tif      (sLen == -1)\tsLen = hLen;\n\telse if (sLen == -2)\tsLen = -2;\n\telse if (sLen < -2)\n\t\t{\n\t\tRSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_PSS, RSA_R_SLEN_CHECK_FAILED);\n\t\tgoto err;\n\t\t}\n\tMSBits = (BN_num_bits(rsa->n) - 1) & 0x7;\n\temLen = RSA_size(rsa);\n\tif (MSBits == 0)\n\t\t{\n\t\t*EM++ = 0;\n\t\temLen--;\n\t\t}\n\tif (sLen == -2)\n\t\t{\n\t\tsLen = emLen - hLen - 2;\n\t\t}\n\telse if (emLen < (hLen + sLen + 2))\n\t\t{\n\t\tRSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_PSS,\n\t\t   RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);\n\t\tgoto err;\n\t\t}\n\tif (sLen > 0)\n\t\t{\n\t\tsalt = OPENSSL_malloc(sLen);\n\t\tif (!salt)\n\t\t\t{\n\t\t\tRSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_PSS,\n\t\t   \t\tERR_R_MALLOC_FAILURE);\n\t\t\tgoto err;\n\t\t\t}\n\t\tif (!RAND_bytes(salt, sLen))\n\t\t\tgoto err;\n\t\t}\n\tmaskedDBLen = emLen - hLen - 1;\n\tH = EM + maskedDBLen;\n\tEVP_MD_CTX_init(&ctx);\n\tEVP_DigestInit_ex(&ctx, Hash, NULL);\n\tEVP_DigestUpdate(&ctx, zeroes, sizeof zeroes);\n\tEVP_DigestUpdate(&ctx, mHash, hLen);\n\tif (sLen)\n\t\tEVP_DigestUpdate(&ctx, salt, sLen);\n\tEVP_DigestFinal(&ctx, H, NULL);\n\tEVP_MD_CTX_cleanup(&ctx);\n\tPKCS1_MGF1(EM, maskedDBLen, H, hLen, Hash);\n\tp = EM;\n\tp += emLen - sLen - hLen - 2;\n\t*p++ ^= 0x1;\n\tif (sLen > 0)\n\t\t{\n\t\tfor (i = 0; i < sLen; i++)\n\t\t\t*p++ ^= salt[i];\n\t\t}\n\tif (MSBits)\n\t\tEM[0] &= 0xFF >> (8 - MSBits);\n\tEM[emLen - 1] = 0xbc;\n\tret = 1;\n\terr:\n\tif (salt)\n\t\tOPENSSL_free(salt);\n\treturn ret;\n\t}', 'size_t EVP_MD_size(const EVP_MD *md)\n\t{\n\tif (!md)\n\t\treturn -1;\n\treturn md->md_size;\n\t}']