crypto/dsa/dsa_gen.c:535: error: BUFFER_OVERRUN_L3
  Offset added: [8, +oo] Size: [0, 536870848] by call to `BN_MONT_CTX_set`.
Showing all 17 steps of the trace


crypto/dsa/dsa_gen.c:415:17: Call
413.   
414.               /* step 4 */
415.               r = BN_is_prime_fasttest_ex(q, DSS_prime_checks, ctx,
                       ^
416.                                           seed_in ? 1 : 0, cb);
417.               if (r > 0)

crypto/bn/bn_prime.c:215:10: Call
  213.       if (mont == NULL)
  214.           goto err;
  215.       if (!BN_MONT_CTX_set(mont, a, ctx))
                  ^
  216.           goto err;
  217.   

crypto/bn/bn_mont.c:247:1: Parameter `*mont->N.d`
    245.   }
    246.   
    247. > int BN_MONT_CTX_set(BN_MONT_CTX *mont, const BIGNUM *mod, BN_CTX *ctx)
    248.   {
    249.       int ret = 0;

crypto/dsa/dsa_gen.c:535:10: Call
533.       } else
534.           h = 1;
535.       if (!BN_MONT_CTX_set(mont, p, ctx))
                ^
536.           goto err;
537.   

crypto/bn/bn_mont.c:247:1: Parameter `*mont->N.d`
  245.   }
  246.   
  247. > int BN_MONT_CTX_set(BN_MONT_CTX *mont, const BIGNUM *mod, BN_CTX *ctx)
  248.   {
  249.       int ret = 0;

crypto/bn/bn_mont.c:259:10: Call
  257.           goto err;
  258.       R = &(mont->RR);            /* grab RR as a temp */
  259.       if (!BN_copy(&(mont->N), mod))
                  ^
  260.           goto err;               /* Set N */
  261.       if (BN_get_flags(mod, BN_FLG_CONSTTIME) != 0)

crypto/bn/bn_lib.c:312:1: Parameter `*a->d`
    310.   }
    311.   
    312. > BIGNUM *BN_copy(BIGNUM *a, const BIGNUM *b)
    313.   {
    314.       bn_check_top(b);

crypto/bn/bn_lib.c:318:9: Call
    316.       if (a == b)
    317.           return a;
    318.       if (bn_wexpand(a, b->top) == NULL)
                   ^
    319.           return NULL;
    320.   

crypto/bn/bn_lib.c:937:1: Parameter `*a->d`
      935.   }
      936.   
      937. > BIGNUM *bn_wexpand(BIGNUM *a, int words)
      938.   {
      939.       return (words <= a->dmax) ? a : bn_expand2(a, words);

crypto/bn/bn_lib.c:939:37: Call
      937.   BIGNUM *bn_wexpand(BIGNUM *a, int words)
      938.   {
      939.       return (words <= a->dmax) ? a : bn_expand2(a, words);
                                                 ^
      940.   }
      941.   

crypto/bn/bn_lib.c:273:1: Parameter `*b->d`
        271.    */
        272.   
        273. > BIGNUM *bn_expand2(BIGNUM *b, int words)
        274.   {
        275.       bn_check_top(b);

crypto/bn/bn_lib.c:278:23: Call
        276.   
        277.       if (words > b->dmax) {
        278.           BN_ULONG *a = bn_expand_internal(b, words);
                                     ^
        279.           if (!a)
        280.               return NULL;

crypto/bn/bn_lib.c:235:1: <Offset trace>
233.   /* This is used by bn_expand2() */
234.   /* The caller MUST check that words > b->dmax before calling this */
235. > static BN_ULONG *bn_expand_internal(const BIGNUM *b, int words)
236.   {
237.       BN_ULONG *a = NULL;

crypto/bn/bn_lib.c:235:1: Parameter `b->top`
          233.   /* This is used by bn_expand2() */
          234.   /* The caller MUST check that words > b->dmax before calling this */
          235. > static BN_ULONG *bn_expand_internal(const BIGNUM *b, int words)
          236.   {
          237.       BN_ULONG *a = NULL;

crypto/bn/bn_lib.c:235:1: <Length trace>
233.   /* This is used by bn_expand2() */
234.   /* The caller MUST check that words > b->dmax before calling this */
235. > static BN_ULONG *bn_expand_internal(const BIGNUM *b, int words)
236.   {
237.       BN_ULONG *a = NULL;

crypto/bn/bn_lib.c:235:1: Parameter `*b->d`
          233.   /* This is used by bn_expand2() */
          234.   /* The caller MUST check that words > b->dmax before calling this */
          235. > static BN_ULONG *bn_expand_internal(const BIGNUM *b, int words)
          236.   {
          237.       BN_ULONG *a = NULL;

crypto/bn/bn_lib.c:260:9: Array access: Offset added: [8, +oo] Size: [0, 536870848] by call to `BN_MONT_CTX_set` 
          258.       assert(b->top <= words);
          259.       if (b->top > 0)
          260.           memcpy(a, b->d, sizeof(*a) * b->top);
                         ^
          261.   
          262.       return a;

static BN_ULONG *bn_expand_internal(const BIGNUM *b, int words)
{
    BN_ULONG *a = NULL;
    bn_check_top(b);
    if (words > (INT_MAX / (4 * BN_BITS2))) {
        BNerr(BN_F_BN_EXPAND_INTERNAL, BN_R_BIGNUM_TOO_LONG);
        return NULL;
    }
    if (BN_get_flags(b, BN_FLG_STATIC_DATA)) {
        BNerr(BN_F_BN_EXPAND_INTERNAL, BN_R_EXPAND_ON_STATIC_BIGNUM_DATA);
        return NULL;
    }
    if (BN_get_flags(b, BN_FLG_SECURE))
        a = OPENSSL_secure_zalloc(words * sizeof(*a));
    else
        a = OPENSSL_zalloc(words * sizeof(*a));
    if (a == NULL) {
        BNerr(BN_F_BN_EXPAND_INTERNAL, ERR_R_MALLOC_FAILURE);
        return NULL;
    }
    assert(b->top <= words);
    if (b->top > 0)
        memcpy(a, b->d, sizeof(*a) * b->top);
    return a;
}
['int dsa_builtin_paramgen2(DSA *ret, size_t L, size_t N,\n                          const EVP_MD *evpmd, const unsigned char *seed_in,\n                          size_t seed_len, int idx, unsigned char *seed_out,\n                          int *counter_ret, unsigned long *h_ret,\n                          BN_GENCB *cb)\n{\n    int ok = -1;\n    unsigned char *seed = NULL, *seed_tmp = NULL;\n    unsigned char md[EVP_MAX_MD_SIZE];\n    int mdsize;\n    BIGNUM *r0, *W, *X, *c, *test;\n    BIGNUM *g = NULL, *q = NULL, *p = NULL;\n    BN_MONT_CTX *mont = NULL;\n    int i, k, n = 0, m = 0, qsize = N >> 3;\n    int counter = 0;\n    int r = 0;\n    BN_CTX *ctx = NULL;\n    EVP_MD_CTX *mctx = EVP_MD_CTX_new();\n    unsigned int h = 2;\n    if (mctx == NULL)\n        goto err;\n    if (evpmd == NULL) {\n        if (N == 160)\n            evpmd = EVP_sha1();\n        else if (N == 224)\n            evpmd = EVP_sha224();\n        else\n            evpmd = EVP_sha256();\n    }\n    mdsize = EVP_MD_size(evpmd);\n    if (!ret->p || !ret->q || idx >= 0) {\n        if (seed_len == 0)\n            seed_len = mdsize;\n        seed = OPENSSL_malloc(seed_len);\n        if (seed_out)\n            seed_tmp = seed_out;\n        else\n            seed_tmp = OPENSSL_malloc(seed_len);\n        if (seed == NULL || seed_tmp == NULL)\n            goto err;\n        if (seed_in)\n            memcpy(seed, seed_in, seed_len);\n    }\n    if ((ctx = BN_CTX_new()) == NULL)\n        goto err;\n    if ((mont = BN_MONT_CTX_new()) == NULL)\n        goto err;\n    BN_CTX_start(ctx);\n    r0 = BN_CTX_get(ctx);\n    g = BN_CTX_get(ctx);\n    W = BN_CTX_get(ctx);\n    X = BN_CTX_get(ctx);\n    c = BN_CTX_get(ctx);\n    test = BN_CTX_get(ctx);\n    if (test == NULL)\n        goto err;\n    if (ret->p && ret->q) {\n        p = ret->p;\n        q = ret->q;\n        if (idx >= 0)\n            memcpy(seed_tmp, seed, seed_len);\n        goto g_only;\n    } else {\n        p = BN_CTX_get(ctx);\n        q = BN_CTX_get(ctx);\n        if (q == NULL)\n            goto err;\n    }\n    if (!BN_lshift(test, BN_value_one(), L - 1))\n        goto err;\n    for (;;) {\n        for (;;) {\n            unsigned char *pmd;\n            if (!BN_GENCB_call(cb, 0, m++))\n                goto err;\n            if (!seed_in) {\n                if (RAND_bytes(seed, seed_len) <= 0)\n                    goto err;\n            }\n            if (!EVP_Digest(seed, seed_len, md, NULL, evpmd, NULL))\n                goto err;\n            if (mdsize > qsize)\n                pmd = md + mdsize - qsize;\n            else\n                pmd = md;\n            if (mdsize < qsize)\n                memset(md + mdsize, 0, qsize - mdsize);\n            pmd[0] |= 0x80;\n            pmd[qsize - 1] |= 0x01;\n            if (!BN_bin2bn(pmd, qsize, q))\n                goto err;\n            r = BN_is_prime_fasttest_ex(q, DSS_prime_checks, ctx,\n                                        seed_in ? 1 : 0, cb);\n            if (r > 0)\n                break;\n            if (r != 0)\n                goto err;\n            if (seed_in) {\n                ok = 0;\n                DSAerr(DSA_F_DSA_BUILTIN_PARAMGEN2, DSA_R_Q_NOT_PRIME);\n                goto err;\n            }\n        }\n        if (seed_out)\n            memcpy(seed_out, seed, seed_len);\n        if (!BN_GENCB_call(cb, 2, 0))\n            goto err;\n        if (!BN_GENCB_call(cb, 3, 0))\n            goto err;\n        counter = 0;\n        n = (L - 1) / (mdsize << 3);\n        for (;;) {\n            if ((counter != 0) && !BN_GENCB_call(cb, 0, counter))\n                goto err;\n            BN_zero(W);\n            for (k = 0; k <= n; k++) {\n                for (i = seed_len - 1; i >= 0; i--) {\n                    seed[i]++;\n                    if (seed[i] != 0)\n                        break;\n                }\n                if (!EVP_Digest(seed, seed_len, md, NULL, evpmd, NULL))\n                    goto err;\n                if (!BN_bin2bn(md, mdsize, r0))\n                    goto err;\n                if (!BN_lshift(r0, r0, (mdsize << 3) * k))\n                    goto err;\n                if (!BN_add(W, W, r0))\n                    goto err;\n            }\n            if (!BN_mask_bits(W, L - 1))\n                goto err;\n            if (!BN_copy(X, W))\n                goto err;\n            if (!BN_add(X, X, test))\n                goto err;\n            if (!BN_lshift1(r0, q))\n                goto err;\n            if (!BN_mod(c, X, r0, ctx))\n                goto err;\n            if (!BN_sub(r0, c, BN_value_one()))\n                goto err;\n            if (!BN_sub(p, X, r0))\n                goto err;\n            if (BN_cmp(p, test) >= 0) {\n                r = BN_is_prime_fasttest_ex(p, DSS_prime_checks, ctx, 1, cb);\n                if (r > 0)\n                    goto end;\n                if (r != 0)\n                    goto err;\n            }\n            counter++;\n            if (counter >= (int)(4 * L))\n                break;\n        }\n        if (seed_in) {\n            ok = 0;\n            DSAerr(DSA_F_DSA_BUILTIN_PARAMGEN2, DSA_R_INVALID_PARAMETERS);\n            goto err;\n        }\n    }\n end:\n    if (!BN_GENCB_call(cb, 2, 1))\n        goto err;\n g_only:\n    if (!BN_sub(test, p, BN_value_one()))\n        goto err;\n    if (!BN_div(r0, NULL, test, q, ctx))\n        goto err;\n    if (idx < 0) {\n        if (!BN_set_word(test, h))\n            goto err;\n    } else\n        h = 1;\n    if (!BN_MONT_CTX_set(mont, p, ctx))\n        goto err;\n    for (;;) {\n        static const unsigned char ggen[4] = { 0x67, 0x67, 0x65, 0x6e };\n        if (idx >= 0) {\n            md[0] = idx & 0xff;\n            md[1] = (h >> 8) & 0xff;\n            md[2] = h & 0xff;\n            if (!EVP_DigestInit_ex(mctx, evpmd, NULL))\n                goto err;\n            if (!EVP_DigestUpdate(mctx, seed_tmp, seed_len))\n                goto err;\n            if (!EVP_DigestUpdate(mctx, ggen, sizeof(ggen)))\n                goto err;\n            if (!EVP_DigestUpdate(mctx, md, 3))\n                goto err;\n            if (!EVP_DigestFinal_ex(mctx, md, NULL))\n                goto err;\n            if (!BN_bin2bn(md, mdsize, test))\n                goto err;\n        }\n        if (!BN_mod_exp_mont(g, test, r0, p, ctx, mont))\n            goto err;\n        if (!BN_is_one(g))\n            break;\n        if (idx < 0 && !BN_add(test, test, BN_value_one()))\n            goto err;\n        h++;\n        if (idx >= 0 && h > 0xffff)\n            goto err;\n    }\n    if (!BN_GENCB_call(cb, 3, 1))\n        goto err;\n    ok = 1;\n err:\n    if (ok == 1) {\n        if (p != ret->p) {\n            BN_free(ret->p);\n            ret->p = BN_dup(p);\n        }\n        if (q != ret->q) {\n            BN_free(ret->q);\n            ret->q = BN_dup(q);\n        }\n        BN_free(ret->g);\n        ret->g = BN_dup(g);\n        if (ret->p == NULL || ret->q == NULL || ret->g == NULL) {\n            ok = -1;\n            goto err;\n        }\n        if (counter_ret != NULL)\n            *counter_ret = counter;\n        if (h_ret != NULL)\n            *h_ret = h;\n    }\n    OPENSSL_free(seed);\n    if (seed_out != seed_tmp)\n        OPENSSL_free(seed_tmp);\n    if (ctx)\n        BN_CTX_end(ctx);\n    BN_CTX_free(ctx);\n    BN_MONT_CTX_free(mont);\n    EVP_MD_CTX_free(mctx);\n    return ok;\n}', 'int BN_is_prime_fasttest_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed,\n                            int do_trial_division, BN_GENCB *cb)\n{\n    int i, j, ret = -1;\n    int k;\n    BN_CTX *ctx = NULL;\n    BIGNUM *A1, *A1_odd, *check;\n    BN_MONT_CTX *mont = NULL;\n    if (BN_cmp(a, BN_value_one()) <= 0)\n        return 0;\n    if (checks == BN_prime_checks)\n        checks = BN_prime_checks_for_size(BN_num_bits(a));\n    if (!BN_is_odd(a))\n        return BN_is_word(a, 2);\n    if (do_trial_division) {\n        for (i = 1; i < NUMPRIMES; i++) {\n            BN_ULONG mod = BN_mod_word(a, primes[i]);\n            if (mod == (BN_ULONG)-1)\n                goto err;\n            if (mod == 0)\n                return BN_is_word(a, primes[i]);\n        }\n        if (!BN_GENCB_call(cb, 1, -1))\n            goto err;\n    }\n    if (ctx_passed != NULL)\n        ctx = ctx_passed;\n    else if ((ctx = BN_CTX_new()) == NULL)\n        goto err;\n    BN_CTX_start(ctx);\n    A1 = BN_CTX_get(ctx);\n    A1_odd = BN_CTX_get(ctx);\n    check = BN_CTX_get(ctx);\n    if (check == NULL)\n        goto err;\n    if (!BN_copy(A1, a))\n        goto err;\n    if (!BN_sub_word(A1, 1))\n        goto err;\n    if (BN_is_zero(A1)) {\n        ret = 0;\n        goto err;\n    }\n    k = 1;\n    while (!BN_is_bit_set(A1, k))\n        k++;\n    if (!BN_rshift(A1_odd, A1, k))\n        goto err;\n    mont = BN_MONT_CTX_new();\n    if (mont == NULL)\n        goto err;\n    if (!BN_MONT_CTX_set(mont, a, ctx))\n        goto err;\n    for (i = 0; i < checks; i++) {\n        if (!BN_priv_rand_range(check, A1))\n            goto err;\n        if (!BN_add_word(check, 1))\n            goto err;\n        j = witness(check, a, A1, A1_odd, k, ctx, mont);\n        if (j == -1)\n            goto err;\n        if (j) {\n            ret = 0;\n            goto err;\n        }\n        if (!BN_GENCB_call(cb, 1, i))\n            goto err;\n    }\n    ret = 1;\n err:\n    if (ctx != NULL) {\n        BN_CTX_end(ctx);\n        if (ctx_passed == NULL)\n            BN_CTX_free(ctx);\n    }\n    BN_MONT_CTX_free(mont);\n    return ret;\n}', 'int BN_MONT_CTX_set(BN_MONT_CTX *mont, const BIGNUM *mod, BN_CTX *ctx)\n{\n    int ret = 0;\n    BIGNUM *Ri, *R;\n    if (BN_is_zero(mod))\n        return 0;\n    BN_CTX_start(ctx);\n    if ((Ri = BN_CTX_get(ctx)) == NULL)\n        goto err;\n    R = &(mont->RR);\n    if (!BN_copy(&(mont->N), mod))\n        goto err;\n    if (BN_get_flags(mod, BN_FLG_CONSTTIME) != 0)\n        BN_set_flags(&(mont->N), BN_FLG_CONSTTIME);\n    mont->N.neg = 0;\n#ifdef MONT_WORD\n    {\n        BIGNUM tmod;\n        BN_ULONG buf[2];\n        bn_init(&tmod);\n        tmod.d = buf;\n        tmod.dmax = 2;\n        tmod.neg = 0;\n        if (BN_get_flags(mod, BN_FLG_CONSTTIME) != 0)\n            BN_set_flags(&tmod, BN_FLG_CONSTTIME);\n        mont->ri = (BN_num_bits(mod) + (BN_BITS2 - 1)) / BN_BITS2 * BN_BITS2;\n# if defined(OPENSSL_BN_ASM_MONT) && (BN_BITS2<=32)\n        BN_zero(R);\n        if (!(BN_set_bit(R, 2 * BN_BITS2)))\n            goto err;\n        tmod.top = 0;\n        if ((buf[0] = mod->d[0]))\n            tmod.top = 1;\n        if ((buf[1] = mod->top > 1 ? mod->d[1] : 0))\n            tmod.top = 2;\n        if ((BN_mod_inverse(Ri, R, &tmod, ctx)) == NULL)\n            goto err;\n        if (!BN_lshift(Ri, Ri, 2 * BN_BITS2))\n            goto err;\n        if (!BN_is_zero(Ri)) {\n            if (!BN_sub_word(Ri, 1))\n                goto err;\n        } else {\n            if (bn_expand(Ri, (int)sizeof(BN_ULONG) * 2) == NULL)\n                goto err;\n            Ri->neg = 0;\n            Ri->d[0] = BN_MASK2;\n            Ri->d[1] = BN_MASK2;\n            Ri->top = 2;\n        }\n        if (!BN_div(Ri, NULL, Ri, &tmod, ctx))\n            goto err;\n        mont->n0[0] = (Ri->top > 0) ? Ri->d[0] : 0;\n        mont->n0[1] = (Ri->top > 1) ? Ri->d[1] : 0;\n# else\n        BN_zero(R);\n        if (!(BN_set_bit(R, BN_BITS2)))\n            goto err;\n        buf[0] = mod->d[0];\n        buf[1] = 0;\n        tmod.top = buf[0] != 0 ? 1 : 0;\n        if ((BN_mod_inverse(Ri, R, &tmod, ctx)) == NULL)\n            goto err;\n        if (!BN_lshift(Ri, Ri, BN_BITS2))\n            goto err;\n        if (!BN_is_zero(Ri)) {\n            if (!BN_sub_word(Ri, 1))\n                goto err;\n        } else {\n            if (!BN_set_word(Ri, BN_MASK2))\n                goto err;\n        }\n        if (!BN_div(Ri, NULL, Ri, &tmod, ctx))\n            goto err;\n        mont->n0[0] = (Ri->top > 0) ? Ri->d[0] : 0;\n        mont->n0[1] = 0;\n# endif\n    }\n#else\n    {\n        mont->ri = BN_num_bits(&mont->N);\n        BN_zero(R);\n        if (!BN_set_bit(R, mont->ri))\n            goto err;\n        if ((BN_mod_inverse(Ri, R, &mont->N, ctx)) == NULL)\n            goto err;\n        if (!BN_lshift(Ri, Ri, mont->ri))\n            goto err;\n        if (!BN_sub_word(Ri, 1))\n            goto err;\n        if (!BN_div(&(mont->Ni), NULL, Ri, &mont->N, ctx))\n            goto err;\n    }\n#endif\n    BN_zero(&(mont->RR));\n    if (!BN_set_bit(&(mont->RR), mont->ri * 2))\n        goto err;\n    if (!BN_mod(&(mont->RR), &(mont->RR), &(mont->N), ctx))\n        goto err;\n    ret = 1;\n err:\n    BN_CTX_end(ctx);\n    return ret;\n}', 'BIGNUM *BN_copy(BIGNUM *a, const BIGNUM *b)\n{\n    bn_check_top(b);\n    if (a == b)\n        return a;\n    if (bn_wexpand(a, b->top) == NULL)\n        return NULL;\n    if (b->top > 0)\n        memcpy(a->d, b->d, sizeof(b->d[0]) * b->top);\n    a->top = b->top;\n    a->neg = b->neg;\n    bn_check_top(a);\n    return a;\n}', 'BIGNUM *bn_wexpand(BIGNUM *a, int words)\n{\n    return (words <= a->dmax) ? a : bn_expand2(a, words);\n}', 'BIGNUM *bn_expand2(BIGNUM *b, int words)\n{\n    bn_check_top(b);\n    if (words > b->dmax) {\n        BN_ULONG *a = bn_expand_internal(b, words);\n        if (!a)\n            return NULL;\n        if (b->d) {\n            OPENSSL_cleanse(b->d, b->dmax * sizeof(b->d[0]));\n            bn_free_d(b);\n        }\n        b->d = a;\n        b->dmax = words;\n    }\n    bn_check_top(b);\n    return b;\n}', 'static BN_ULONG *bn_expand_internal(const BIGNUM *b, int words)\n{\n    BN_ULONG *a = NULL;\n    bn_check_top(b);\n    if (words > (INT_MAX / (4 * BN_BITS2))) {\n        BNerr(BN_F_BN_EXPAND_INTERNAL, BN_R_BIGNUM_TOO_LONG);\n        return NULL;\n    }\n    if (BN_get_flags(b, BN_FLG_STATIC_DATA)) {\n        BNerr(BN_F_BN_EXPAND_INTERNAL, BN_R_EXPAND_ON_STATIC_BIGNUM_DATA);\n        return NULL;\n    }\n    if (BN_get_flags(b, BN_FLG_SECURE))\n        a = OPENSSL_secure_zalloc(words * sizeof(*a));\n    else\n        a = OPENSSL_zalloc(words * sizeof(*a));\n    if (a == NULL) {\n        BNerr(BN_F_BN_EXPAND_INTERNAL, ERR_R_MALLOC_FAILURE);\n        return NULL;\n    }\n    assert(b->top <= words);\n    if (b->top > 0)\n        memcpy(a, b->d, sizeof(*a) * b->top);\n    return a;\n}']