crypto/rsa/rsa_pmeth.c:258: error: BUFFER_OVERRUN_L3
  Offset: [-1, +oo] (⇐ [0, 1] + [-1, +oo]) Size: [1, +oo] by call to `RSA_verify_PKCS1_PSS_mgf1`.
Showing all 8 steps of the trace


crypto/rsa/rsa_pmeth.c:252:18: Call
250.           } else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
251.               int ret;
252.               if (!setup_tbuf(rctx, ctx))
                        ^
253.                   return -1;
254.               ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,

crypto/rsa/rsa_pmeth.c:97:1: Parameter `*ctx->tbuf`
  95.   }
  96.   
  97. > static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
  98.   {
  99.       if (ctx->tbuf != NULL)

crypto/rsa/rsa_pmeth.c:258:19: Call
256.               if (ret <= 0)
257.                   return 0;
258.               ret = RSA_verify_PKCS1_PSS_mgf1(rsa, tbs,
                         ^
259.                                               rctx->md, rctx->mgf1md,
260.                                               rctx->tbuf, rctx->saltlen);

crypto/rsa/rsa_pss.c:32:1: <Offset trace>
30.   }
31.   
32. > int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
33.                                 const EVP_MD *Hash, const EVP_MD *mgf1Hash,
34.                                 const unsigned char *EM, int sLen)

crypto/rsa/rsa_pss.c:32:1: Parameter `sLen`
  30.   }
  31.   
  32. > int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
  33.                                 const EVP_MD *Hash, const EVP_MD *mgf1Hash,
  34.                                 const unsigned char *EM, int sLen)

crypto/rsa/rsa_pss.c:32:1: <Length trace>
30.   }
31.   
32. > int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
33.                                 const EVP_MD *Hash, const EVP_MD *mgf1Hash,
34.                                 const unsigned char *EM, int sLen)

crypto/rsa/rsa_pss.c:32:1: Parameter `*EM`
  30.   }
  31.   
  32. > int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
  33.                                 const EVP_MD *Hash, const EVP_MD *mgf1Hash,
  34.                                 const unsigned char *EM, int sLen)

crypto/rsa/rsa_pss.c:83:9: Array access: Offset: [-1, +oo] (⇐ [0, 1] + [-1, +oo]) Size: [1, +oo] by call to `RSA_verify_PKCS1_PSS_mgf1` 
  81.           goto err;
  82.       }
  83.       if (EM[emLen - 1] != 0xbc) {
                ^
  84.           RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_LAST_OCTET_INVALID);
  85.           goto err;

int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
                              const EVP_MD *Hash, const EVP_MD *mgf1Hash,
                              const unsigned char *EM, int sLen)
{
    int i;
    int ret = 0;
    int hLen, maskedDBLen, MSBits, emLen;
    const unsigned char *H;
    unsigned char *DB = NULL;
    EVP_MD_CTX *ctx = EVP_MD_CTX_new();
    unsigned char H_[EVP_MAX_MD_SIZE];
    if (ctx == NULL)
        goto err;
    if (mgf1Hash == NULL)
        mgf1Hash = Hash;
    hLen = EVP_MD_size(Hash);
    if (hLen < 0)
        goto err;
    if (sLen == -1)
        sLen = hLen;
    else if (sLen == -2)
        sLen = -2;
    else if (sLen < -2) {
        RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_SLEN_CHECK_FAILED);
        goto err;
    }
    MSBits = (BN_num_bits(rsa->n) - 1) & 0x7;
    emLen = RSA_size(rsa);
    if (EM[0] & (0xFF << MSBits)) {
        RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_FIRST_OCTET_INVALID);
        goto err;
    }
    if (MSBits == 0) {
        EM++;
        emLen--;
    }
    if (emLen < (hLen + sLen + 2)) {
        RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_DATA_TOO_LARGE);
        goto err;
    }
    if (EM[emLen - 1] != 0xbc) {
        RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_LAST_OCTET_INVALID);
        goto err;
    }
    maskedDBLen = emLen - hLen - 1;
    H = EM + maskedDBLen;
    DB = OPENSSL_malloc(maskedDBLen);
    if (DB == NULL) {
        RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, ERR_R_MALLOC_FAILURE);
        goto err;
    }
    if (PKCS1_MGF1(DB, maskedDBLen, H, hLen, mgf1Hash) < 0)
        goto err;
    for (i = 0; i < maskedDBLen; i++)
        DB[i] ^= EM[i];
    if (MSBits)
        DB[0] &= 0xFF >> (8 - MSBits);
    for (i = 0; DB[i] == 0 && i < (maskedDBLen - 1); i++) ;
    if (DB[i++] != 0x1) {
        RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_SLEN_RECOVERY_FAILED);
        goto err;
    }
    if (sLen >= 0 && (maskedDBLen - i) != sLen) {
        RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_SLEN_CHECK_FAILED);
        goto err;
    }
    if (!EVP_DigestInit_ex(ctx, Hash, NULL)
        || !EVP_DigestUpdate(ctx, zeroes, sizeof zeroes)
        || !EVP_DigestUpdate(ctx, mHash, hLen))
        goto err;
    if (maskedDBLen - i) {
        if (!EVP_DigestUpdate(ctx, DB + i, maskedDBLen - i))
            goto err;
    }
    if (!EVP_DigestFinal_ex(ctx, H_, NULL))
        goto err;
    if (memcmp(H_, H, hLen)) {
        RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_BAD_SIGNATURE);
        ret = 0;
    } else
        ret = 1;
 err:
    OPENSSL_free(DB);
    EVP_MD_CTX_free(ctx);
    return ret;
}
['static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,\n                           const unsigned char *sig, size_t siglen,\n                           const unsigned char *tbs, size_t tbslen)\n{\n    RSA_PKEY_CTX *rctx = ctx->data;\n    RSA *rsa = ctx->pkey->pkey.rsa;\n    size_t rslen;\n    if (rctx->md) {\n        if (rctx->pad_mode == RSA_PKCS1_PADDING)\n            return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen,\n                              sig, siglen, rsa);\n        if (tbslen != (size_t)EVP_MD_size(rctx->md)) {\n            RSAerr(RSA_F_PKEY_RSA_VERIFY, RSA_R_INVALID_DIGEST_LENGTH);\n            return -1;\n        }\n        if (rctx->pad_mode == RSA_X931_PADDING) {\n            if (pkey_rsa_verifyrecover(ctx, NULL, &rslen, sig, siglen) <= 0)\n                return 0;\n        } else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {\n            int ret;\n            if (!setup_tbuf(rctx, ctx))\n                return -1;\n            ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,\n                                     rsa, RSA_NO_PADDING);\n            if (ret <= 0)\n                return 0;\n            ret = RSA_verify_PKCS1_PSS_mgf1(rsa, tbs,\n                                            rctx->md, rctx->mgf1md,\n                                            rctx->tbuf, rctx->saltlen);\n            if (ret <= 0)\n                return 0;\n            return 1;\n        } else\n            return -1;\n    } else {\n        if (!setup_tbuf(rctx, ctx))\n            return -1;\n        rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf,\n                                   rsa, rctx->pad_mode);\n        if (rslen == 0)\n            return 0;\n    }\n    if ((rslen != tbslen) || memcmp(tbs, rctx->tbuf, rslen))\n        return 0;\n    return 1;\n}', 'static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)\n{\n    if (ctx->tbuf != NULL)\n        return 1;\n    ctx->tbuf = OPENSSL_malloc(EVP_PKEY_size(pk->pkey));\n    if (ctx->tbuf == NULL)\n        return 0;\n    return 1;\n}', 'int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,\n                              const EVP_MD *Hash, const EVP_MD *mgf1Hash,\n                              const unsigned char *EM, int sLen)\n{\n    int i;\n    int ret = 0;\n    int hLen, maskedDBLen, MSBits, emLen;\n    const unsigned char *H;\n    unsigned char *DB = NULL;\n    EVP_MD_CTX *ctx = EVP_MD_CTX_new();\n    unsigned char H_[EVP_MAX_MD_SIZE];\n    if (ctx == NULL)\n        goto err;\n    if (mgf1Hash == NULL)\n        mgf1Hash = Hash;\n    hLen = EVP_MD_size(Hash);\n    if (hLen < 0)\n        goto err;\n    if (sLen == -1)\n        sLen = hLen;\n    else if (sLen == -2)\n        sLen = -2;\n    else if (sLen < -2) {\n        RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_SLEN_CHECK_FAILED);\n        goto err;\n    }\n    MSBits = (BN_num_bits(rsa->n) - 1) & 0x7;\n    emLen = RSA_size(rsa);\n    if (EM[0] & (0xFF << MSBits)) {\n        RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_FIRST_OCTET_INVALID);\n        goto err;\n    }\n    if (MSBits == 0) {\n        EM++;\n        emLen--;\n    }\n    if (emLen < (hLen + sLen + 2)) {\n        RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_DATA_TOO_LARGE);\n        goto err;\n    }\n    if (EM[emLen - 1] != 0xbc) {\n        RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_LAST_OCTET_INVALID);\n        goto err;\n    }\n    maskedDBLen = emLen - hLen - 1;\n    H = EM + maskedDBLen;\n    DB = OPENSSL_malloc(maskedDBLen);\n    if (DB == NULL) {\n        RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, ERR_R_MALLOC_FAILURE);\n        goto err;\n    }\n    if (PKCS1_MGF1(DB, maskedDBLen, H, hLen, mgf1Hash) < 0)\n        goto err;\n    for (i = 0; i < maskedDBLen; i++)\n        DB[i] ^= EM[i];\n    if (MSBits)\n        DB[0] &= 0xFF >> (8 - MSBits);\n    for (i = 0; DB[i] == 0 && i < (maskedDBLen - 1); i++) ;\n    if (DB[i++] != 0x1) {\n        RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_SLEN_RECOVERY_FAILED);\n        goto err;\n    }\n    if (sLen >= 0 && (maskedDBLen - i) != sLen) {\n        RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_SLEN_CHECK_FAILED);\n        goto err;\n    }\n    if (!EVP_DigestInit_ex(ctx, Hash, NULL)\n        || !EVP_DigestUpdate(ctx, zeroes, sizeof zeroes)\n        || !EVP_DigestUpdate(ctx, mHash, hLen))\n        goto err;\n    if (maskedDBLen - i) {\n        if (!EVP_DigestUpdate(ctx, DB + i, maskedDBLen - i))\n            goto err;\n    }\n    if (!EVP_DigestFinal_ex(ctx, H_, NULL))\n        goto err;\n    if (memcmp(H_, H, hLen)) {\n        RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_BAD_SIGNATURE);\n        ret = 0;\n    } else\n        ret = 1;\n err:\n    OPENSSL_free(DB);\n    EVP_MD_CTX_free(ctx);\n    return ret;\n}']