ssl/s3_cbc.c:600: error: INTEGER_OVERFLOW_L2
  ([3, +oo] - [2, 6]):unsigned32.
Showing all 6 steps of the trace


ssl/s3_cbc.c:599:41: <LHS trace>
597.        * least two because the header is larger than a single block.
598.        */
599.       if (num_blocks > variance_blocks + (is_sslv3 ? 1 : 0)) {
                                               ^
600.           num_starting_blocks = num_blocks - variance_blocks;
601.           k = md_block_size * num_starting_blocks;

ssl/s3_cbc.c:599:41: Assignment
597.        * least two because the header is larger than a single block.
598.        */
599.       if (num_blocks > variance_blocks + (is_sslv3 ? 1 : 0)) {
                                               ^
600.           num_starting_blocks = num_blocks - variance_blocks;
601.           k = md_block_size * num_starting_blocks;

ssl/s3_cbc.c:541:23: <RHS trace>
539.        * variance_blocks can be reduced.
540.        */
541.       variance_blocks = is_sslv3 ? 2 : 6;
                             ^
542.       /*
543.        * From now on we're dealing with the MAC, which conceptually has 13

ssl/s3_cbc.c:541:23: Assignment
539.        * variance_blocks can be reduced.
540.        */
541.       variance_blocks = is_sslv3 ? 2 : 6;
                             ^
542.       /*
543.        * From now on we're dealing with the MAC, which conceptually has 13

ssl/s3_cbc.c:541:5: Assignment
539.        * variance_blocks can be reduced.
540.        */
541.       variance_blocks = is_sslv3 ? 2 : 6;
           ^
542.       /*
543.        * From now on we're dealing with the MAC, which conceptually has 13

ssl/s3_cbc.c:600:9: Binary operation: ([3, +oo] - [2, 6]):unsigned32
598.        */
599.       if (num_blocks > variance_blocks + (is_sslv3 ? 1 : 0)) {
600.           num_starting_blocks = num_blocks - variance_blocks;
               ^
601.           k = md_block_size * num_starting_blocks;
602.       }

void ssl3_cbc_digest_record(const EVP_MD_CTX *ctx,
                            unsigned char *md_out,
                            size_t *md_out_size,
                            const unsigned char header[13],
                            const unsigned char *data,
                            size_t data_plus_mac_size,
                            size_t data_plus_mac_plus_padding_size,
                            const unsigned char *mac_secret,
                            unsigned mac_secret_length, char is_sslv3)
{
    union {
        double align;
        unsigned char c[sizeof(LARGEST_DIGEST_CTX)];
    } md_state;
    void (*md_final_raw) (void *ctx, unsigned char *md_out);
    void (*md_transform) (void *ctx, const unsigned char *block);
    unsigned md_size, md_block_size = 64;
    unsigned sslv3_pad_length = 40, header_length, variance_blocks,
        len, max_mac_bytes, num_blocks,
        num_starting_blocks, k, mac_end_offset, c, index_a, index_b;
    unsigned int bits;
    unsigned char length_bytes[MAX_HASH_BIT_COUNT_BYTES];
    unsigned char hmac_pad[MAX_HASH_BLOCK_SIZE];
    unsigned char first_block[MAX_HASH_BLOCK_SIZE];
    unsigned char mac_out[EVP_MAX_MD_SIZE];
    unsigned i, j, md_out_size_u;
    EVP_MD_CTX md_ctx;
    unsigned md_length_size = 8;
    char length_is_big_endian = 1;
    int ret;
    OPENSSL_assert(data_plus_mac_plus_padding_size < 1024 * 1024);
    switch (EVP_MD_CTX_type(ctx)) {
    case NID_md5:
        MD5_Init((MD5_CTX *)md_state.c);
        md_final_raw = tls1_md5_final_raw;
        md_transform =
            (void (*)(void *ctx, const unsigned char *block))MD5_Transform;
        md_size = 16;
        sslv3_pad_length = 48;
        length_is_big_endian = 0;
        break;
    case NID_sha1:
        SHA1_Init((SHA_CTX *)md_state.c);
        md_final_raw = tls1_sha1_final_raw;
        md_transform =
            (void (*)(void *ctx, const unsigned char *block))SHA1_Transform;
        md_size = 20;
        break;
#ifndef OPENSSL_NO_SHA256
    case NID_sha224:
        SHA224_Init((SHA256_CTX *)md_state.c);
        md_final_raw = tls1_sha256_final_raw;
        md_transform =
            (void (*)(void *ctx, const unsigned char *block))SHA256_Transform;
        md_size = 224 / 8;
        break;
    case NID_sha256:
        SHA256_Init((SHA256_CTX *)md_state.c);
        md_final_raw = tls1_sha256_final_raw;
        md_transform =
            (void (*)(void *ctx, const unsigned char *block))SHA256_Transform;
        md_size = 32;
        break;
#endif
#ifndef OPENSSL_NO_SHA512
    case NID_sha384:
        SHA384_Init((SHA512_CTX *)md_state.c);
        md_final_raw = tls1_sha512_final_raw;
        md_transform =
            (void (*)(void *ctx, const unsigned char *block))SHA512_Transform;
        md_size = 384 / 8;
        md_block_size = 128;
        md_length_size = 16;
        break;
    case NID_sha512:
        SHA512_Init((SHA512_CTX *)md_state.c);
        md_final_raw = tls1_sha512_final_raw;
        md_transform =
            (void (*)(void *ctx, const unsigned char *block))SHA512_Transform;
        md_size = 64;
        md_block_size = 128;
        md_length_size = 16;
        break;
#endif
    default:
        OPENSSL_assert(0);
        if (md_out_size)
            *md_out_size = -1;
        return;
    }
    OPENSSL_assert(md_length_size <= MAX_HASH_BIT_COUNT_BYTES);
    OPENSSL_assert(md_block_size <= MAX_HASH_BLOCK_SIZE);
    OPENSSL_assert(md_size <= EVP_MAX_MD_SIZE);
    header_length = 13;
    if (is_sslv3) {
        header_length = mac_secret_length + sslv3_pad_length + 8    +
            1    +
            2   ;
    }
    variance_blocks = is_sslv3 ? 2 : 6;
    len = data_plus_mac_plus_padding_size + header_length;
    max_mac_bytes = len - md_size - 1;
    num_blocks =
        (max_mac_bytes + 1 + md_length_size + md_block_size -
         1) / md_block_size;
    num_starting_blocks = 0;
    k = 0;
    mac_end_offset = data_plus_mac_size + header_length - md_size;
    c = mac_end_offset % md_block_size;
    index_a = mac_end_offset / md_block_size;
    index_b = (mac_end_offset + md_length_size) / md_block_size;
    if (num_blocks > variance_blocks + (is_sslv3 ? 1 : 0)) {
        num_starting_blocks = num_blocks - variance_blocks;
        k = md_block_size * num_starting_blocks;
    }
    bits = 8 * mac_end_offset;
    if (!is_sslv3) {
        bits += 8 * md_block_size;
        memset(hmac_pad, 0, md_block_size);
        OPENSSL_assert(mac_secret_length <= sizeof(hmac_pad));
        memcpy(hmac_pad, mac_secret, mac_secret_length);
        for (i = 0; i < md_block_size; i++)
            hmac_pad[i] ^= 0x36;
        md_transform(md_state.c, hmac_pad);
    }
    if (length_is_big_endian) {
        memset(length_bytes, 0, md_length_size - 4);
        length_bytes[md_length_size - 4] = (unsigned char)(bits >> 24);
        length_bytes[md_length_size - 3] = (unsigned char)(bits >> 16);
        length_bytes[md_length_size - 2] = (unsigned char)(bits >> 8);
        length_bytes[md_length_size - 1] = (unsigned char)bits;
    } else {
        memset(length_bytes, 0, md_length_size);
        length_bytes[md_length_size - 5] = (unsigned char)(bits >> 24);
        length_bytes[md_length_size - 6] = (unsigned char)(bits >> 16);
        length_bytes[md_length_size - 7] = (unsigned char)(bits >> 8);
        length_bytes[md_length_size - 8] = (unsigned char)bits;
    }
    if (k > 0) {
        if (is_sslv3) {
            unsigned overhang = header_length - md_block_size;
            md_transform(md_state.c, header);
            memcpy(first_block, header + md_block_size, overhang);
            memcpy(first_block + overhang, data, md_block_size - overhang);
            md_transform(md_state.c, first_block);
            for (i = 1; i < k / md_block_size - 1; i++)
                md_transform(md_state.c, data + md_block_size * i - overhang);
        } else {
            memcpy(first_block, header, 13);
            memcpy(first_block + 13, data, md_block_size - 13);
            md_transform(md_state.c, first_block);
            for (i = 1; i < k / md_block_size; i++)
                md_transform(md_state.c, data + md_block_size * i - 13);
        }
    }
    memset(mac_out, 0, sizeof(mac_out));
    for (i = num_starting_blocks; i <= num_starting_blocks + variance_blocks;
         i++) {
        unsigned char block[MAX_HASH_BLOCK_SIZE];
        unsigned char is_block_a = constant_time_eq_8(i, index_a);
        unsigned char is_block_b = constant_time_eq_8(i, index_b);
        for (j = 0; j < md_block_size; j++) {
            unsigned char b = 0, is_past_c, is_past_cp1;
            if (k < header_length)
                b = header[k];
            else if (k < data_plus_mac_plus_padding_size + header_length)
                b = data[k - header_length];
            k++;
            is_past_c = is_block_a & constant_time_ge_8(j, c);
            is_past_cp1 = is_block_a & constant_time_ge_8(j, c + 1);
            b = constant_time_select_8(is_past_c, 0x80, b);
            b = b & ~is_past_cp1;
            b &= ~is_block_b | is_block_a;
            if (j >= md_block_size - md_length_size) {
                b = constant_time_select_8(is_block_b,
                                           length_bytes[j -
                                                        (md_block_size -
                                                         md_length_size)], b);
            }
            block[j] = b;
        }
        md_transform(md_state.c, block);
        md_final_raw(md_state.c, block);
        for (j = 0; j < md_size; j++)
            mac_out[j] |= block[j] & is_block_b;
    }
    EVP_MD_CTX_init(&md_ctx);
    EVP_DigestInit_ex(&md_ctx, ctx->digest, NULL   );
    if (is_sslv3) {
        memset(hmac_pad, 0x5c, sslv3_pad_length);
        EVP_DigestUpdate(&md_ctx, mac_secret, mac_secret_length);
        EVP_DigestUpdate(&md_ctx, hmac_pad, sslv3_pad_length);
        EVP_DigestUpdate(&md_ctx, mac_out, md_size);
    } else {
        for (i = 0; i < md_block_size; i++)
            hmac_pad[i] ^= 0x6a;
        EVP_DigestUpdate(&md_ctx, hmac_pad, md_block_size);
        EVP_DigestUpdate(&md_ctx, mac_out, md_size);
    }
    ret = EVP_DigestFinal(&md_ctx, md_out, &md_out_size_u);
    if (ret && md_out_size)
        *md_out_size = md_out_size_u;
    EVP_MD_CTX_cleanup(&md_ctx);
}
['void ssl3_cbc_digest_record(const EVP_MD_CTX *ctx,\n                            unsigned char *md_out,\n                            size_t *md_out_size,\n                            const unsigned char header[13],\n                            const unsigned char *data,\n                            size_t data_plus_mac_size,\n                            size_t data_plus_mac_plus_padding_size,\n                            const unsigned char *mac_secret,\n                            unsigned mac_secret_length, char is_sslv3)\n{\n    union {\n        double align;\n        unsigned char c[sizeof(LARGEST_DIGEST_CTX)];\n    } md_state;\n    void (*md_final_raw) (void *ctx, unsigned char *md_out);\n    void (*md_transform) (void *ctx, const unsigned char *block);\n    unsigned md_size, md_block_size = 64;\n    unsigned sslv3_pad_length = 40, header_length, variance_blocks,\n        len, max_mac_bytes, num_blocks,\n        num_starting_blocks, k, mac_end_offset, c, index_a, index_b;\n    unsigned int bits;\n    unsigned char length_bytes[MAX_HASH_BIT_COUNT_BYTES];\n    unsigned char hmac_pad[MAX_HASH_BLOCK_SIZE];\n    unsigned char first_block[MAX_HASH_BLOCK_SIZE];\n    unsigned char mac_out[EVP_MAX_MD_SIZE];\n    unsigned i, j, md_out_size_u;\n    EVP_MD_CTX md_ctx;\n    unsigned md_length_size = 8;\n    char length_is_big_endian = 1;\n    int ret;\n    OPENSSL_assert(data_plus_mac_plus_padding_size < 1024 * 1024);\n    switch (EVP_MD_CTX_type(ctx)) {\n    case NID_md5:\n        MD5_Init((MD5_CTX *)md_state.c);\n        md_final_raw = tls1_md5_final_raw;\n        md_transform =\n            (void (*)(void *ctx, const unsigned char *block))MD5_Transform;\n        md_size = 16;\n        sslv3_pad_length = 48;\n        length_is_big_endian = 0;\n        break;\n    case NID_sha1:\n        SHA1_Init((SHA_CTX *)md_state.c);\n        md_final_raw = tls1_sha1_final_raw;\n        md_transform =\n            (void (*)(void *ctx, const unsigned char *block))SHA1_Transform;\n        md_size = 20;\n        break;\n#ifndef OPENSSL_NO_SHA256\n    case NID_sha224:\n        SHA224_Init((SHA256_CTX *)md_state.c);\n        md_final_raw = tls1_sha256_final_raw;\n        md_transform =\n            (void (*)(void *ctx, const unsigned char *block))SHA256_Transform;\n        md_size = 224 / 8;\n        break;\n    case NID_sha256:\n        SHA256_Init((SHA256_CTX *)md_state.c);\n        md_final_raw = tls1_sha256_final_raw;\n        md_transform =\n            (void (*)(void *ctx, const unsigned char *block))SHA256_Transform;\n        md_size = 32;\n        break;\n#endif\n#ifndef OPENSSL_NO_SHA512\n    case NID_sha384:\n        SHA384_Init((SHA512_CTX *)md_state.c);\n        md_final_raw = tls1_sha512_final_raw;\n        md_transform =\n            (void (*)(void *ctx, const unsigned char *block))SHA512_Transform;\n        md_size = 384 / 8;\n        md_block_size = 128;\n        md_length_size = 16;\n        break;\n    case NID_sha512:\n        SHA512_Init((SHA512_CTX *)md_state.c);\n        md_final_raw = tls1_sha512_final_raw;\n        md_transform =\n            (void (*)(void *ctx, const unsigned char *block))SHA512_Transform;\n        md_size = 64;\n        md_block_size = 128;\n        md_length_size = 16;\n        break;\n#endif\n    default:\n        OPENSSL_assert(0);\n        if (md_out_size)\n            *md_out_size = -1;\n        return;\n    }\n    OPENSSL_assert(md_length_size <= MAX_HASH_BIT_COUNT_BYTES);\n    OPENSSL_assert(md_block_size <= MAX_HASH_BLOCK_SIZE);\n    OPENSSL_assert(md_size <= EVP_MAX_MD_SIZE);\n    header_length = 13;\n    if (is_sslv3) {\n        header_length = mac_secret_length + sslv3_pad_length + 8    +\n            1    +\n            2   ;\n    }\n    variance_blocks = is_sslv3 ? 2 : 6;\n    len = data_plus_mac_plus_padding_size + header_length;\n    max_mac_bytes = len - md_size - 1;\n    num_blocks =\n        (max_mac_bytes + 1 + md_length_size + md_block_size -\n         1) / md_block_size;\n    num_starting_blocks = 0;\n    k = 0;\n    mac_end_offset = data_plus_mac_size + header_length - md_size;\n    c = mac_end_offset % md_block_size;\n    index_a = mac_end_offset / md_block_size;\n    index_b = (mac_end_offset + md_length_size) / md_block_size;\n    if (num_blocks > variance_blocks + (is_sslv3 ? 1 : 0)) {\n        num_starting_blocks = num_blocks - variance_blocks;\n        k = md_block_size * num_starting_blocks;\n    }\n    bits = 8 * mac_end_offset;\n    if (!is_sslv3) {\n        bits += 8 * md_block_size;\n        memset(hmac_pad, 0, md_block_size);\n        OPENSSL_assert(mac_secret_length <= sizeof(hmac_pad));\n        memcpy(hmac_pad, mac_secret, mac_secret_length);\n        for (i = 0; i < md_block_size; i++)\n            hmac_pad[i] ^= 0x36;\n        md_transform(md_state.c, hmac_pad);\n    }\n    if (length_is_big_endian) {\n        memset(length_bytes, 0, md_length_size - 4);\n        length_bytes[md_length_size - 4] = (unsigned char)(bits >> 24);\n        length_bytes[md_length_size - 3] = (unsigned char)(bits >> 16);\n        length_bytes[md_length_size - 2] = (unsigned char)(bits >> 8);\n        length_bytes[md_length_size - 1] = (unsigned char)bits;\n    } else {\n        memset(length_bytes, 0, md_length_size);\n        length_bytes[md_length_size - 5] = (unsigned char)(bits >> 24);\n        length_bytes[md_length_size - 6] = (unsigned char)(bits >> 16);\n        length_bytes[md_length_size - 7] = (unsigned char)(bits >> 8);\n        length_bytes[md_length_size - 8] = (unsigned char)bits;\n    }\n    if (k > 0) {\n        if (is_sslv3) {\n            unsigned overhang = header_length - md_block_size;\n            md_transform(md_state.c, header);\n            memcpy(first_block, header + md_block_size, overhang);\n            memcpy(first_block + overhang, data, md_block_size - overhang);\n            md_transform(md_state.c, first_block);\n            for (i = 1; i < k / md_block_size - 1; i++)\n                md_transform(md_state.c, data + md_block_size * i - overhang);\n        } else {\n            memcpy(first_block, header, 13);\n            memcpy(first_block + 13, data, md_block_size - 13);\n            md_transform(md_state.c, first_block);\n            for (i = 1; i < k / md_block_size; i++)\n                md_transform(md_state.c, data + md_block_size * i - 13);\n        }\n    }\n    memset(mac_out, 0, sizeof(mac_out));\n    for (i = num_starting_blocks; i <= num_starting_blocks + variance_blocks;\n         i++) {\n        unsigned char block[MAX_HASH_BLOCK_SIZE];\n        unsigned char is_block_a = constant_time_eq_8(i, index_a);\n        unsigned char is_block_b = constant_time_eq_8(i, index_b);\n        for (j = 0; j < md_block_size; j++) {\n            unsigned char b = 0, is_past_c, is_past_cp1;\n            if (k < header_length)\n                b = header[k];\n            else if (k < data_plus_mac_plus_padding_size + header_length)\n                b = data[k - header_length];\n            k++;\n            is_past_c = is_block_a & constant_time_ge_8(j, c);\n            is_past_cp1 = is_block_a & constant_time_ge_8(j, c + 1);\n            b = constant_time_select_8(is_past_c, 0x80, b);\n            b = b & ~is_past_cp1;\n            b &= ~is_block_b | is_block_a;\n            if (j >= md_block_size - md_length_size) {\n                b = constant_time_select_8(is_block_b,\n                                           length_bytes[j -\n                                                        (md_block_size -\n                                                         md_length_size)], b);\n            }\n            block[j] = b;\n        }\n        md_transform(md_state.c, block);\n        md_final_raw(md_state.c, block);\n        for (j = 0; j < md_size; j++)\n            mac_out[j] |= block[j] & is_block_b;\n    }\n    EVP_MD_CTX_init(&md_ctx);\n    EVP_DigestInit_ex(&md_ctx, ctx->digest, NULL   );\n    if (is_sslv3) {\n        memset(hmac_pad, 0x5c, sslv3_pad_length);\n        EVP_DigestUpdate(&md_ctx, mac_secret, mac_secret_length);\n        EVP_DigestUpdate(&md_ctx, hmac_pad, sslv3_pad_length);\n        EVP_DigestUpdate(&md_ctx, mac_out, md_size);\n    } else {\n        for (i = 0; i < md_block_size; i++)\n            hmac_pad[i] ^= 0x6a;\n        EVP_DigestUpdate(&md_ctx, hmac_pad, md_block_size);\n        EVP_DigestUpdate(&md_ctx, mac_out, md_size);\n    }\n    ret = EVP_DigestFinal(&md_ctx, md_out, &md_out_size_u);\n    if (ret && md_out_size)\n        *md_out_size = md_out_size_u;\n    EVP_MD_CTX_cleanup(&md_ctx);\n}']