test/ecdsatest.c:375: error: INTEGER_OVERFLOW_L2
  ([-1, +oo] - 1):unsigned32.
Showing all 5 steps of the trace


test/ecdsatest.c:344:19: <LHS trace>
342.           (void)BIO_flush(out);
343.           /* create signature */
344.           sig_len = ECDSA_size(eckey);
                         ^
345.           if ((signature = OPENSSL_malloc(sig_len)) == NULL)
346.               goto builtin_err;

test/ecdsatest.c:344:19: Call
342.           (void)BIO_flush(out);
343.           /* create signature */
344.           sig_len = ECDSA_size(eckey);
                         ^
345.           if ((signature = OPENSSL_malloc(sig_len)) == NULL)
346.               goto builtin_err;

crypto/ec/ec_asn1.c:1219:9: Assignment
  1217.   
  1218.       if (r == NULL)
  1219.           return 0;
                  ^
  1220.       group = EC_KEY_get0_group(r);
  1221.       if (group == NULL)

test/ecdsatest.c:344:9: Assignment
342.           (void)BIO_flush(out);
343.           /* create signature */
344.           sig_len = ECDSA_size(eckey);
               ^
345.           if ((signature = OPENSSL_malloc(sig_len)) == NULL)
346.               goto builtin_err;

test/ecdsatest.c:375:13: Binary operation: ([-1, +oo] - 1):unsigned32
373.           (void)BIO_flush(out);
374.           /* wrong length */
375.           if (ECDSA_verify(0, digest, 20, signature, sig_len - 1, eckey) == 1) {
                   ^
376.               BIO_printf(out, " failed\n");
377.               goto builtin_err;

int test_builtin(BIO *out)
{
    EC_builtin_curve *curves = NULL;
    size_t crv_len = 0, n = 0;
    EC_KEY *eckey = NULL, *wrong_eckey = NULL;
    EC_GROUP *group;
    ECDSA_SIG *ecdsa_sig = NULL, *modified_sig = NULL;
    unsigned char digest[20], wrong_digest[20];
    unsigned char *signature = NULL;
    const unsigned char *sig_ptr;
    unsigned char *sig_ptr2;
    unsigned char *raw_buf = NULL;
    const BIGNUM *sig_r, *sig_s;
    BIGNUM *modified_r = NULL, *modified_s = NULL;
    BIGNUM *unmodified_r = NULL, *unmodified_s = NULL;
    unsigned int sig_len, degree, r_len, s_len, bn_len, buf_len;
    int nid, ret = 0;
    if (RAND_bytes(digest, 20) <= 0 || RAND_bytes(wrong_digest, 20) <= 0) {
        BIO_printf(out, "ERROR: unable to get random data\n");
        goto builtin_err;
    }
    BIO_printf(out, "\ntesting ECDSA_sign() and ECDSA_verify() "
               "with some internal curves:\n");
    crv_len = EC_get_builtin_curves(NULL, 0);
    curves = OPENSSL_malloc(sizeof(*curves) * crv_len);
    if (curves == NULL) {
        BIO_printf(out, "malloc error\n");
        goto builtin_err;
    }
    if (!EC_get_builtin_curves(curves, crv_len)) {
        BIO_printf(out, "unable to get internal curves\n");
        goto builtin_err;
    }
    for (n = 0; n < crv_len; n++) {
        unsigned char dirt, offset;
        nid = curves[n].nid;
        if (nid == NID_ipsec4 || nid == NID_X25519)
            continue;
        if ((eckey = EC_KEY_new()) == NULL)
            goto builtin_err;
        group = EC_GROUP_new_by_curve_name(nid);
        if (group == NULL)
            goto builtin_err;
        if (EC_KEY_set_group(eckey, group) == 0)
            goto builtin_err;
        EC_GROUP_free(group);
        degree = EC_GROUP_get_degree(EC_KEY_get0_group(eckey));
        if (degree < 160) {
            EC_KEY_free(eckey);
            eckey = NULL;
            continue;
        }
        BIO_printf(out, "%s: ", OBJ_nid2sn(nid));
        if (!EC_KEY_generate_key(eckey)) {
            BIO_printf(out, " failed\n");
            goto builtin_err;
        }
        if ((wrong_eckey = EC_KEY_new()) == NULL)
            goto builtin_err;
        group = EC_GROUP_new_by_curve_name(nid);
        if (group == NULL)
            goto builtin_err;
        if (EC_KEY_set_group(wrong_eckey, group) == 0)
            goto builtin_err;
        EC_GROUP_free(group);
        if (!EC_KEY_generate_key(wrong_eckey)) {
            BIO_printf(out, " failed\n");
            goto builtin_err;
        }
        BIO_printf(out, ".");
        (void)BIO_flush(out);
        if (!EC_KEY_check_key(eckey)) {
            BIO_printf(out, " failed\n");
            goto builtin_err;
        }
        BIO_printf(out, ".");
        (void)BIO_flush(out);
        sig_len = ECDSA_size(eckey);
        if ((signature = OPENSSL_malloc(sig_len)) == NULL)
            goto builtin_err;
        if (!ECDSA_sign(0, digest, 20, signature, &sig_len, eckey)) {
            BIO_printf(out, " failed\n");
            goto builtin_err;
        }
        BIO_printf(out, ".");
        (void)BIO_flush(out);
        if (ECDSA_verify(0, digest, 20, signature, sig_len, eckey) != 1) {
            BIO_printf(out, " failed\n");
            goto builtin_err;
        }
        BIO_printf(out, ".");
        (void)BIO_flush(out);
        if (ECDSA_verify(0, digest, 20, signature, sig_len, wrong_eckey) == 1) {
            BIO_printf(out, " failed\n");
            goto builtin_err;
        }
        BIO_printf(out, ".");
        (void)BIO_flush(out);
        if (ECDSA_verify(0, wrong_digest, 20, signature, sig_len, eckey) == 1) {
            BIO_printf(out, " failed\n");
            goto builtin_err;
        }
        BIO_printf(out, ".");
        (void)BIO_flush(out);
        if (ECDSA_verify(0, digest, 20, signature, sig_len - 1, eckey) == 1) {
            BIO_printf(out, " failed\n");
            goto builtin_err;
        }
        BIO_printf(out, ".");
        (void)BIO_flush(out);
        sig_ptr = signature;
        if ((ecdsa_sig = d2i_ECDSA_SIG(NULL, &sig_ptr, sig_len)) == NULL) {
            BIO_printf(out, " failed\n");
            goto builtin_err;
        }
        ECDSA_SIG_get0(ecdsa_sig, &sig_r, &sig_s);
        r_len = BN_num_bytes(sig_r);
        s_len = BN_num_bytes(sig_s);
        bn_len = (degree + 7) / 8;
        if ((r_len > bn_len) || (s_len > bn_len)) {
            BIO_printf(out, " failed\n");
            goto builtin_err;
        }
        buf_len = 2 * bn_len;
        if ((raw_buf = OPENSSL_zalloc(buf_len)) == NULL)
            goto builtin_err;
        BN_bn2bin(sig_r, raw_buf + bn_len - r_len);
        BN_bn2bin(sig_s, raw_buf + buf_len - s_len);
        offset = raw_buf[10] % buf_len;
        dirt = raw_buf[11] ? raw_buf[11] : 1;
        raw_buf[offset] ^= dirt;
        modified_sig = ECDSA_SIG_new();
        if (modified_sig == NULL)
            goto builtin_err;
        if (((modified_r = BN_bin2bn(raw_buf, bn_len, NULL)) == NULL)
            || ((modified_s = BN_bin2bn(raw_buf + bn_len, bn_len, NULL)) == NULL)
            || !ECDSA_SIG_set0(modified_sig, modified_r, modified_s)) {
            BN_free(modified_r);
            BN_free(modified_s);
            goto builtin_err;
        }
        sig_ptr2 = signature;
        sig_len = i2d_ECDSA_SIG(modified_sig, &sig_ptr2);
        if (ECDSA_verify(0, digest, 20, signature, sig_len, eckey) == 1) {
            BIO_printf(out, " failed\n");
            goto builtin_err;
        }
        raw_buf[offset] ^= dirt;
        if (((unmodified_r = BN_bin2bn(raw_buf, bn_len, NULL)) == NULL)
            || ((unmodified_s = BN_bin2bn(raw_buf + bn_len, bn_len, NULL)) == NULL)
            || !ECDSA_SIG_set0(modified_sig, unmodified_r, unmodified_s)) {
            BN_free(unmodified_r);
            BN_free(unmodified_s);
            goto builtin_err;
        }
        sig_ptr2 = signature;
        sig_len = i2d_ECDSA_SIG(modified_sig, &sig_ptr2);
        if (ECDSA_verify(0, digest, 20, signature, sig_len, eckey) != 1) {
            BIO_printf(out, " failed\n");
            goto builtin_err;
        }
        BIO_printf(out, ".");
        (void)BIO_flush(out);
        BIO_printf(out, " ok\n");
        ERR_clear_error();
        OPENSSL_free(signature);
        signature = NULL;
        EC_KEY_free(eckey);
        eckey = NULL;
        EC_KEY_free(wrong_eckey);
        wrong_eckey = NULL;
        ECDSA_SIG_free(ecdsa_sig);
        ecdsa_sig = NULL;
        ECDSA_SIG_free(modified_sig);
        modified_sig = NULL;
        OPENSSL_free(raw_buf);
        raw_buf = NULL;
    }
    ret = 1;
 builtin_err:
    EC_KEY_free(eckey);
    EC_KEY_free(wrong_eckey);
    ECDSA_SIG_free(ecdsa_sig);
    ECDSA_SIG_free(modified_sig);
    OPENSSL_free(signature);
    OPENSSL_free(raw_buf);
    OPENSSL_free(curves);
    return ret;
}
['int test_builtin(BIO *out)\n{\n    EC_builtin_curve *curves = NULL;\n    size_t crv_len = 0, n = 0;\n    EC_KEY *eckey = NULL, *wrong_eckey = NULL;\n    EC_GROUP *group;\n    ECDSA_SIG *ecdsa_sig = NULL, *modified_sig = NULL;\n    unsigned char digest[20], wrong_digest[20];\n    unsigned char *signature = NULL;\n    const unsigned char *sig_ptr;\n    unsigned char *sig_ptr2;\n    unsigned char *raw_buf = NULL;\n    const BIGNUM *sig_r, *sig_s;\n    BIGNUM *modified_r = NULL, *modified_s = NULL;\n    BIGNUM *unmodified_r = NULL, *unmodified_s = NULL;\n    unsigned int sig_len, degree, r_len, s_len, bn_len, buf_len;\n    int nid, ret = 0;\n    if (RAND_bytes(digest, 20) <= 0 || RAND_bytes(wrong_digest, 20) <= 0) {\n        BIO_printf(out, "ERROR: unable to get random data\\n");\n        goto builtin_err;\n    }\n    BIO_printf(out, "\\ntesting ECDSA_sign() and ECDSA_verify() "\n               "with some internal curves:\\n");\n    crv_len = EC_get_builtin_curves(NULL, 0);\n    curves = OPENSSL_malloc(sizeof(*curves) * crv_len);\n    if (curves == NULL) {\n        BIO_printf(out, "malloc error\\n");\n        goto builtin_err;\n    }\n    if (!EC_get_builtin_curves(curves, crv_len)) {\n        BIO_printf(out, "unable to get internal curves\\n");\n        goto builtin_err;\n    }\n    for (n = 0; n < crv_len; n++) {\n        unsigned char dirt, offset;\n        nid = curves[n].nid;\n        if (nid == NID_ipsec4 || nid == NID_X25519)\n            continue;\n        if ((eckey = EC_KEY_new()) == NULL)\n            goto builtin_err;\n        group = EC_GROUP_new_by_curve_name(nid);\n        if (group == NULL)\n            goto builtin_err;\n        if (EC_KEY_set_group(eckey, group) == 0)\n            goto builtin_err;\n        EC_GROUP_free(group);\n        degree = EC_GROUP_get_degree(EC_KEY_get0_group(eckey));\n        if (degree < 160) {\n            EC_KEY_free(eckey);\n            eckey = NULL;\n            continue;\n        }\n        BIO_printf(out, "%s: ", OBJ_nid2sn(nid));\n        if (!EC_KEY_generate_key(eckey)) {\n            BIO_printf(out, " failed\\n");\n            goto builtin_err;\n        }\n        if ((wrong_eckey = EC_KEY_new()) == NULL)\n            goto builtin_err;\n        group = EC_GROUP_new_by_curve_name(nid);\n        if (group == NULL)\n            goto builtin_err;\n        if (EC_KEY_set_group(wrong_eckey, group) == 0)\n            goto builtin_err;\n        EC_GROUP_free(group);\n        if (!EC_KEY_generate_key(wrong_eckey)) {\n            BIO_printf(out, " failed\\n");\n            goto builtin_err;\n        }\n        BIO_printf(out, ".");\n        (void)BIO_flush(out);\n        if (!EC_KEY_check_key(eckey)) {\n            BIO_printf(out, " failed\\n");\n            goto builtin_err;\n        }\n        BIO_printf(out, ".");\n        (void)BIO_flush(out);\n        sig_len = ECDSA_size(eckey);\n        if ((signature = OPENSSL_malloc(sig_len)) == NULL)\n            goto builtin_err;\n        if (!ECDSA_sign(0, digest, 20, signature, &sig_len, eckey)) {\n            BIO_printf(out, " failed\\n");\n            goto builtin_err;\n        }\n        BIO_printf(out, ".");\n        (void)BIO_flush(out);\n        if (ECDSA_verify(0, digest, 20, signature, sig_len, eckey) != 1) {\n            BIO_printf(out, " failed\\n");\n            goto builtin_err;\n        }\n        BIO_printf(out, ".");\n        (void)BIO_flush(out);\n        if (ECDSA_verify(0, digest, 20, signature, sig_len, wrong_eckey) == 1) {\n            BIO_printf(out, " failed\\n");\n            goto builtin_err;\n        }\n        BIO_printf(out, ".");\n        (void)BIO_flush(out);\n        if (ECDSA_verify(0, wrong_digest, 20, signature, sig_len, eckey) == 1) {\n            BIO_printf(out, " failed\\n");\n            goto builtin_err;\n        }\n        BIO_printf(out, ".");\n        (void)BIO_flush(out);\n        if (ECDSA_verify(0, digest, 20, signature, sig_len - 1, eckey) == 1) {\n            BIO_printf(out, " failed\\n");\n            goto builtin_err;\n        }\n        BIO_printf(out, ".");\n        (void)BIO_flush(out);\n        sig_ptr = signature;\n        if ((ecdsa_sig = d2i_ECDSA_SIG(NULL, &sig_ptr, sig_len)) == NULL) {\n            BIO_printf(out, " failed\\n");\n            goto builtin_err;\n        }\n        ECDSA_SIG_get0(ecdsa_sig, &sig_r, &sig_s);\n        r_len = BN_num_bytes(sig_r);\n        s_len = BN_num_bytes(sig_s);\n        bn_len = (degree + 7) / 8;\n        if ((r_len > bn_len) || (s_len > bn_len)) {\n            BIO_printf(out, " failed\\n");\n            goto builtin_err;\n        }\n        buf_len = 2 * bn_len;\n        if ((raw_buf = OPENSSL_zalloc(buf_len)) == NULL)\n            goto builtin_err;\n        BN_bn2bin(sig_r, raw_buf + bn_len - r_len);\n        BN_bn2bin(sig_s, raw_buf + buf_len - s_len);\n        offset = raw_buf[10] % buf_len;\n        dirt = raw_buf[11] ? raw_buf[11] : 1;\n        raw_buf[offset] ^= dirt;\n        modified_sig = ECDSA_SIG_new();\n        if (modified_sig == NULL)\n            goto builtin_err;\n        if (((modified_r = BN_bin2bn(raw_buf, bn_len, NULL)) == NULL)\n            || ((modified_s = BN_bin2bn(raw_buf + bn_len, bn_len, NULL)) == NULL)\n            || !ECDSA_SIG_set0(modified_sig, modified_r, modified_s)) {\n            BN_free(modified_r);\n            BN_free(modified_s);\n            goto builtin_err;\n        }\n        sig_ptr2 = signature;\n        sig_len = i2d_ECDSA_SIG(modified_sig, &sig_ptr2);\n        if (ECDSA_verify(0, digest, 20, signature, sig_len, eckey) == 1) {\n            BIO_printf(out, " failed\\n");\n            goto builtin_err;\n        }\n        raw_buf[offset] ^= dirt;\n        if (((unmodified_r = BN_bin2bn(raw_buf, bn_len, NULL)) == NULL)\n            || ((unmodified_s = BN_bin2bn(raw_buf + bn_len, bn_len, NULL)) == NULL)\n            || !ECDSA_SIG_set0(modified_sig, unmodified_r, unmodified_s)) {\n            BN_free(unmodified_r);\n            BN_free(unmodified_s);\n            goto builtin_err;\n        }\n        sig_ptr2 = signature;\n        sig_len = i2d_ECDSA_SIG(modified_sig, &sig_ptr2);\n        if (ECDSA_verify(0, digest, 20, signature, sig_len, eckey) != 1) {\n            BIO_printf(out, " failed\\n");\n            goto builtin_err;\n        }\n        BIO_printf(out, ".");\n        (void)BIO_flush(out);\n        BIO_printf(out, " ok\\n");\n        ERR_clear_error();\n        OPENSSL_free(signature);\n        signature = NULL;\n        EC_KEY_free(eckey);\n        eckey = NULL;\n        EC_KEY_free(wrong_eckey);\n        wrong_eckey = NULL;\n        ECDSA_SIG_free(ecdsa_sig);\n        ecdsa_sig = NULL;\n        ECDSA_SIG_free(modified_sig);\n        modified_sig = NULL;\n        OPENSSL_free(raw_buf);\n        raw_buf = NULL;\n    }\n    ret = 1;\n builtin_err:\n    EC_KEY_free(eckey);\n    EC_KEY_free(wrong_eckey);\n    ECDSA_SIG_free(ecdsa_sig);\n    ECDSA_SIG_free(modified_sig);\n    OPENSSL_free(signature);\n    OPENSSL_free(raw_buf);\n    OPENSSL_free(curves);\n    return ret;\n}', 'int ECDSA_size(const EC_KEY *r)\n{\n    int ret, i;\n    ASN1_INTEGER bs;\n    unsigned char buf[4];\n    const EC_GROUP *group;\n    if (r == NULL)\n        return 0;\n    group = EC_KEY_get0_group(r);\n    if (group == NULL)\n        return 0;\n    i = EC_GROUP_order_bits(group);\n    if (i == 0)\n        return 0;\n    bs.length = (i + 7) / 8;\n    bs.data = buf;\n    bs.type = V_ASN1_INTEGER;\n    buf[0] = 0xff;\n    i = i2d_ASN1_INTEGER(&bs, NULL);\n    i += i;\n    ret = ASN1_object_size(1, i, V_ASN1_SEQUENCE);\n    return (ret);\n}']