test/ecdsatest.c:299: error: INTEGER_OVERFLOW_L2
  ([-1, +oo] - 1):unsigned32.
Showing all 5 steps of the trace


test/ecdsatest.c:274:19: <LHS trace>
272.   
273.           /* create signature */
274.           sig_len = ECDSA_size(eckey);
                         ^
275.           if (!TEST_ptr(signature = OPENSSL_malloc(sig_len))
276.                   || !TEST_true(ECDSA_sign(0, digest, SHA512_DIGEST_LENGTH,

test/ecdsatest.c:274:19: Call
272.   
273.           /* create signature */
274.           sig_len = ECDSA_size(eckey);
                         ^
275.           if (!TEST_ptr(signature = OPENSSL_malloc(sig_len))
276.                   || !TEST_true(ECDSA_sign(0, digest, SHA512_DIGEST_LENGTH,

crypto/ec/ec_asn1.c:1203:9: Assignment
  1201.   
  1202.       if (r == NULL)
  1203.           return 0;
                  ^
  1204.       group = EC_KEY_get0_group(r);
  1205.       if (group == NULL)

test/ecdsatest.c:274:9: Assignment
272.   
273.           /* create signature */
274.           sig_len = ECDSA_size(eckey);
               ^
275.           if (!TEST_ptr(signature = OPENSSL_malloc(sig_len))
276.                   || !TEST_true(ECDSA_sign(0, digest, SHA512_DIGEST_LENGTH,

test/ecdsatest.c:299:14: Binary operation: ([-1, +oo] - 1):unsigned32
297.   
298.           /* wrong length */
299.           if (!TEST_int_ne(ECDSA_verify(0, digest, SHA512_DIGEST_LENGTH,
                    ^
300.                                         signature, sig_len - 1, eckey),
301.                            1))

static int test_builtin(void)
{
    EC_builtin_curve *curves = NULL;
    size_t crv_len = 0, n = 0;
    EC_KEY *eckey = NULL, *wrong_eckey = NULL;
    EC_GROUP *group;
    ECDSA_SIG *ecdsa_sig = NULL, *modified_sig = NULL;
    unsigned char digest[SHA512_DIGEST_LENGTH];
    unsigned char wrong_digest[SHA512_DIGEST_LENGTH];
    unsigned char *signature = NULL;
    const unsigned char *sig_ptr;
    unsigned char *sig_ptr2;
    unsigned char *raw_buf = NULL;
    const BIGNUM *sig_r, *sig_s;
    BIGNUM *modified_r = NULL, *modified_s = NULL;
    BIGNUM *unmodified_r = NULL, *unmodified_s = NULL;
    unsigned int sig_len, degree, r_len, s_len, bn_len, buf_len;
    int nid, ret = 0;
    if (!TEST_true(RAND_bytes(digest, SHA512_DIGEST_LENGTH))
            || !TEST_true(RAND_bytes(wrong_digest, SHA512_DIGEST_LENGTH)))
        goto builtin_err;
    crv_len = EC_get_builtin_curves(NULL, 0);
    if (!TEST_ptr(curves = OPENSSL_malloc(sizeof(*curves) * crv_len))
            || !TEST_true(EC_get_builtin_curves(curves, crv_len)))
        goto builtin_err;
    for (n = 0; n < crv_len; n++) {
        unsigned char dirt, offset;
        nid = curves[n].nid;
        if (nid == NID_ipsec4 || nid == NID_ipsec3)
            continue;
        if (!TEST_ptr(eckey = EC_KEY_new())
                || !TEST_ptr(group = EC_GROUP_new_by_curve_name(nid))
                || !TEST_true(EC_KEY_set_group(eckey, group)))
            goto builtin_err;
        EC_GROUP_free(group);
        degree = EC_GROUP_get_degree(EC_KEY_get0_group(eckey));
        TEST_info("testing %s", OBJ_nid2sn(nid));
        if (!TEST_true(EC_KEY_generate_key(eckey)))
            goto builtin_err;
        if (!TEST_ptr(wrong_eckey = EC_KEY_new())
                || !TEST_ptr(group = EC_GROUP_new_by_curve_name(nid))
                || !TEST_true(EC_KEY_set_group(wrong_eckey, group)))
            goto builtin_err;
        EC_GROUP_free(group);
        if (!TEST_true(EC_KEY_generate_key(wrong_eckey)))
            goto builtin_err;
        if (!TEST_true(EC_KEY_check_key(eckey)))
            goto builtin_err;
        sig_len = ECDSA_size(eckey);
        if (!TEST_ptr(signature = OPENSSL_malloc(sig_len))
                || !TEST_true(ECDSA_sign(0, digest, SHA512_DIGEST_LENGTH,
                                         signature, &sig_len, eckey)))
            goto builtin_err;
        if (!TEST_int_eq(ECDSA_verify(0, digest, SHA512_DIGEST_LENGTH,
                                      signature, sig_len, eckey),
                         1))
            goto builtin_err;
        if (!TEST_int_ne(ECDSA_verify(0, digest, SHA512_DIGEST_LENGTH,
                                      signature, sig_len, wrong_eckey),
                         1))
            goto builtin_err;
        if (!TEST_int_ne(ECDSA_verify(0, wrong_digest, SHA512_DIGEST_LENGTH,
                                      signature, sig_len, eckey),
                         1))
            goto builtin_err;
        if (!TEST_int_ne(ECDSA_verify(0, digest, SHA512_DIGEST_LENGTH,
                                      signature, sig_len - 1, eckey),
                         1))
            goto builtin_err;
        sig_ptr = signature;
        if (!TEST_ptr(ecdsa_sig = d2i_ECDSA_SIG(NULL, &sig_ptr, sig_len)))
            goto builtin_err;
        ECDSA_SIG_get0(ecdsa_sig, &sig_r, &sig_s);
        r_len = BN_num_bytes(sig_r);
        s_len = BN_num_bytes(sig_s);
        bn_len = (degree + 7) / 8;
        if (!TEST_false(r_len > bn_len)
                || !TEST_false(s_len > bn_len))
            goto builtin_err;
        buf_len = 2 * bn_len;
        if (!TEST_ptr(raw_buf = OPENSSL_zalloc(buf_len)))
            goto builtin_err;
        BN_bn2bin(sig_r, raw_buf + bn_len - r_len);
        BN_bn2bin(sig_s, raw_buf + buf_len - s_len);
        offset = raw_buf[10] % buf_len;
        dirt = raw_buf[11] ? raw_buf[11] : 1;
        raw_buf[offset] ^= dirt;
        if (!TEST_ptr(modified_sig = ECDSA_SIG_new()))
            goto builtin_err;
        if (!TEST_ptr(modified_r = BN_bin2bn(raw_buf, bn_len, NULL))
                || !TEST_ptr(modified_s = BN_bin2bn(raw_buf + bn_len,
                                                    bn_len, NULL))
                || !TEST_true(ECDSA_SIG_set0(modified_sig,
                                             modified_r, modified_s))) {
            BN_free(modified_r);
            BN_free(modified_s);
            goto builtin_err;
        }
        sig_ptr2 = signature;
        sig_len = i2d_ECDSA_SIG(modified_sig, &sig_ptr2);
        if (!TEST_false(ECDSA_verify(0, digest, SHA512_DIGEST_LENGTH,
                                     signature, sig_len, eckey)))
            goto builtin_err;
        raw_buf[offset] ^= dirt;
        if (!TEST_ptr(unmodified_r = BN_bin2bn(raw_buf, bn_len, NULL))
                || !TEST_ptr(unmodified_s = BN_bin2bn(raw_buf + bn_len,
                                                      bn_len, NULL))
                || !TEST_true(ECDSA_SIG_set0(modified_sig, unmodified_r,
                                             unmodified_s))) {
            BN_free(unmodified_r);
            BN_free(unmodified_s);
            goto builtin_err;
        }
        sig_ptr2 = signature;
        sig_len = i2d_ECDSA_SIG(modified_sig, &sig_ptr2);
        if (!TEST_true(ECDSA_verify(0, digest, SHA512_DIGEST_LENGTH,
                                    signature, sig_len, eckey)))
            goto builtin_err;
        ERR_clear_error();
        OPENSSL_free(signature);
        signature = NULL;
        EC_KEY_free(eckey);
        eckey = NULL;
        EC_KEY_free(wrong_eckey);
        wrong_eckey = NULL;
        ECDSA_SIG_free(ecdsa_sig);
        ecdsa_sig = NULL;
        ECDSA_SIG_free(modified_sig);
        modified_sig = NULL;
        OPENSSL_free(raw_buf);
        raw_buf = NULL;
    }
    ret = 1;
 builtin_err:
    EC_KEY_free(eckey);
    EC_KEY_free(wrong_eckey);
    ECDSA_SIG_free(ecdsa_sig);
    ECDSA_SIG_free(modified_sig);
    OPENSSL_free(signature);
    OPENSSL_free(raw_buf);
    OPENSSL_free(curves);
    return ret;
}
['static int test_builtin(void)\n{\n    EC_builtin_curve *curves = NULL;\n    size_t crv_len = 0, n = 0;\n    EC_KEY *eckey = NULL, *wrong_eckey = NULL;\n    EC_GROUP *group;\n    ECDSA_SIG *ecdsa_sig = NULL, *modified_sig = NULL;\n    unsigned char digest[SHA512_DIGEST_LENGTH];\n    unsigned char wrong_digest[SHA512_DIGEST_LENGTH];\n    unsigned char *signature = NULL;\n    const unsigned char *sig_ptr;\n    unsigned char *sig_ptr2;\n    unsigned char *raw_buf = NULL;\n    const BIGNUM *sig_r, *sig_s;\n    BIGNUM *modified_r = NULL, *modified_s = NULL;\n    BIGNUM *unmodified_r = NULL, *unmodified_s = NULL;\n    unsigned int sig_len, degree, r_len, s_len, bn_len, buf_len;\n    int nid, ret = 0;\n    if (!TEST_true(RAND_bytes(digest, SHA512_DIGEST_LENGTH))\n            || !TEST_true(RAND_bytes(wrong_digest, SHA512_DIGEST_LENGTH)))\n        goto builtin_err;\n    crv_len = EC_get_builtin_curves(NULL, 0);\n    if (!TEST_ptr(curves = OPENSSL_malloc(sizeof(*curves) * crv_len))\n            || !TEST_true(EC_get_builtin_curves(curves, crv_len)))\n        goto builtin_err;\n    for (n = 0; n < crv_len; n++) {\n        unsigned char dirt, offset;\n        nid = curves[n].nid;\n        if (nid == NID_ipsec4 || nid == NID_ipsec3)\n            continue;\n        if (!TEST_ptr(eckey = EC_KEY_new())\n                || !TEST_ptr(group = EC_GROUP_new_by_curve_name(nid))\n                || !TEST_true(EC_KEY_set_group(eckey, group)))\n            goto builtin_err;\n        EC_GROUP_free(group);\n        degree = EC_GROUP_get_degree(EC_KEY_get0_group(eckey));\n        TEST_info("testing %s", OBJ_nid2sn(nid));\n        if (!TEST_true(EC_KEY_generate_key(eckey)))\n            goto builtin_err;\n        if (!TEST_ptr(wrong_eckey = EC_KEY_new())\n                || !TEST_ptr(group = EC_GROUP_new_by_curve_name(nid))\n                || !TEST_true(EC_KEY_set_group(wrong_eckey, group)))\n            goto builtin_err;\n        EC_GROUP_free(group);\n        if (!TEST_true(EC_KEY_generate_key(wrong_eckey)))\n            goto builtin_err;\n        if (!TEST_true(EC_KEY_check_key(eckey)))\n            goto builtin_err;\n        sig_len = ECDSA_size(eckey);\n        if (!TEST_ptr(signature = OPENSSL_malloc(sig_len))\n                || !TEST_true(ECDSA_sign(0, digest, SHA512_DIGEST_LENGTH,\n                                         signature, &sig_len, eckey)))\n            goto builtin_err;\n        if (!TEST_int_eq(ECDSA_verify(0, digest, SHA512_DIGEST_LENGTH,\n                                      signature, sig_len, eckey),\n                         1))\n            goto builtin_err;\n        if (!TEST_int_ne(ECDSA_verify(0, digest, SHA512_DIGEST_LENGTH,\n                                      signature, sig_len, wrong_eckey),\n                         1))\n            goto builtin_err;\n        if (!TEST_int_ne(ECDSA_verify(0, wrong_digest, SHA512_DIGEST_LENGTH,\n                                      signature, sig_len, eckey),\n                         1))\n            goto builtin_err;\n        if (!TEST_int_ne(ECDSA_verify(0, digest, SHA512_DIGEST_LENGTH,\n                                      signature, sig_len - 1, eckey),\n                         1))\n            goto builtin_err;\n        sig_ptr = signature;\n        if (!TEST_ptr(ecdsa_sig = d2i_ECDSA_SIG(NULL, &sig_ptr, sig_len)))\n            goto builtin_err;\n        ECDSA_SIG_get0(ecdsa_sig, &sig_r, &sig_s);\n        r_len = BN_num_bytes(sig_r);\n        s_len = BN_num_bytes(sig_s);\n        bn_len = (degree + 7) / 8;\n        if (!TEST_false(r_len > bn_len)\n                || !TEST_false(s_len > bn_len))\n            goto builtin_err;\n        buf_len = 2 * bn_len;\n        if (!TEST_ptr(raw_buf = OPENSSL_zalloc(buf_len)))\n            goto builtin_err;\n        BN_bn2bin(sig_r, raw_buf + bn_len - r_len);\n        BN_bn2bin(sig_s, raw_buf + buf_len - s_len);\n        offset = raw_buf[10] % buf_len;\n        dirt = raw_buf[11] ? raw_buf[11] : 1;\n        raw_buf[offset] ^= dirt;\n        if (!TEST_ptr(modified_sig = ECDSA_SIG_new()))\n            goto builtin_err;\n        if (!TEST_ptr(modified_r = BN_bin2bn(raw_buf, bn_len, NULL))\n                || !TEST_ptr(modified_s = BN_bin2bn(raw_buf + bn_len,\n                                                    bn_len, NULL))\n                || !TEST_true(ECDSA_SIG_set0(modified_sig,\n                                             modified_r, modified_s))) {\n            BN_free(modified_r);\n            BN_free(modified_s);\n            goto builtin_err;\n        }\n        sig_ptr2 = signature;\n        sig_len = i2d_ECDSA_SIG(modified_sig, &sig_ptr2);\n        if (!TEST_false(ECDSA_verify(0, digest, SHA512_DIGEST_LENGTH,\n                                     signature, sig_len, eckey)))\n            goto builtin_err;\n        raw_buf[offset] ^= dirt;\n        if (!TEST_ptr(unmodified_r = BN_bin2bn(raw_buf, bn_len, NULL))\n                || !TEST_ptr(unmodified_s = BN_bin2bn(raw_buf + bn_len,\n                                                      bn_len, NULL))\n                || !TEST_true(ECDSA_SIG_set0(modified_sig, unmodified_r,\n                                             unmodified_s))) {\n            BN_free(unmodified_r);\n            BN_free(unmodified_s);\n            goto builtin_err;\n        }\n        sig_ptr2 = signature;\n        sig_len = i2d_ECDSA_SIG(modified_sig, &sig_ptr2);\n        if (!TEST_true(ECDSA_verify(0, digest, SHA512_DIGEST_LENGTH,\n                                    signature, sig_len, eckey)))\n            goto builtin_err;\n        ERR_clear_error();\n        OPENSSL_free(signature);\n        signature = NULL;\n        EC_KEY_free(eckey);\n        eckey = NULL;\n        EC_KEY_free(wrong_eckey);\n        wrong_eckey = NULL;\n        ECDSA_SIG_free(ecdsa_sig);\n        ecdsa_sig = NULL;\n        ECDSA_SIG_free(modified_sig);\n        modified_sig = NULL;\n        OPENSSL_free(raw_buf);\n        raw_buf = NULL;\n    }\n    ret = 1;\n builtin_err:\n    EC_KEY_free(eckey);\n    EC_KEY_free(wrong_eckey);\n    ECDSA_SIG_free(ecdsa_sig);\n    ECDSA_SIG_free(modified_sig);\n    OPENSSL_free(signature);\n    OPENSSL_free(raw_buf);\n    OPENSSL_free(curves);\n    return ret;\n}', 'int ECDSA_size(const EC_KEY *r)\n{\n    int ret, i;\n    ASN1_INTEGER bs;\n    unsigned char buf[4];\n    const EC_GROUP *group;\n    if (r == NULL)\n        return 0;\n    group = EC_KEY_get0_group(r);\n    if (group == NULL)\n        return 0;\n    i = EC_GROUP_order_bits(group);\n    if (i == 0)\n        return 0;\n    bs.length = (i + 7) / 8;\n    bs.data = buf;\n    bs.type = V_ASN1_INTEGER;\n    buf[0] = 0xff;\n    i = i2d_ASN1_INTEGER(&bs, NULL);\n    i += i;\n    ret = ASN1_object_size(1, i, V_ASN1_SEQUENCE);\n    return ret;\n}']