crypto/rsa/rsa_sp800_56b_check.c:133: error: BUFFER_OVERRUN_L3
  Offset: [-1, +oo] Size: [1, +oo] by call to `rsa_check_prime_factor_range`.
Showing all 13 steps of the trace


crypto/rsa/rsa_sp800_56b_check.c:131:9: Call
129.   
130.       /* (Steps 5 a-b) prime test */
131.       if (BN_is_prime_fasttest_ex(p, checks, ctx, 1, NULL) != 1
               ^
132.               /* (Step 5c) (√2)(2^(nbits/2 - 1) <= p <= 2^(nbits/2 - 1) */
133.               || rsa_check_prime_factor_range(p, nbits, ctx) != 1)

crypto/bn/bn_prime.c:186:1: Parameter `ctx_passed->stack.depth`
  184.   
  185.   /* See FIPS 186-4 C.3.1 Miller Rabin Probabilistic Primality Test. */
  186. > int BN_is_prime_fasttest_ex(const BIGNUM *w, int checks, BN_CTX *ctx_passed,
  187.                               int do_trial_division, BN_GENCB *cb)
  188.   {

crypto/rsa/rsa_sp800_56b_check.c:133:16: Call
131.       if (BN_is_prime_fasttest_ex(p, checks, ctx, 1, NULL) != 1
132.               /* (Step 5c) (√2)(2^(nbits/2 - 1) <= p <= 2^(nbits/2 - 1) */
133.               || rsa_check_prime_factor_range(p, nbits, ctx) != 1)
                      ^
134.           return 0;
135.   

crypto/rsa/rsa_sp800_56b_check.c:95:5: Call
  93.           return 0;
  94.   
  95.       BN_CTX_start(ctx);
            ^
  96.       tmp = BN_CTX_get(ctx);
  97.       low = BN_CTX_get(ctx);

crypto/bn/bn_ctx.c:171:1: Parameter `*ctx->stack.indexes`
    169.   }
    170.   
    171. > void BN_CTX_start(BN_CTX *ctx)
    172.   {
    173.       CTXDBG("ENTER BN_CTX_start()", ctx);

crypto/rsa/rsa_sp800_56b_check.c:113:5: Call
  111.       ret = 1;
  112.   err:
  113.       BN_CTX_end(ctx);
             ^
  114.       return ret;
  115.   }

crypto/bn/bn_ctx.c:185:1: Parameter `*ctx->stack.indexes`
    183.   }
    184.   
    185. > void BN_CTX_end(BN_CTX *ctx)
    186.   {
    187.       if (ctx == NULL)

crypto/bn/bn_ctx.c:193:27: Call
    191.           ctx->err_stack--;
    192.       else {
    193.           unsigned int fp = BN_STACK_pop(&ctx->stack);
                                     ^
    194.           /* Does this stack frame have anything to release? */
    195.           if (fp < ctx->used)

crypto/bn/bn_ctx.c:268:1: <Offset trace>
266.   }
267.   
268. > static unsigned int BN_STACK_pop(BN_STACK *st)
269.   {
270.       return st->indexes[--(st->depth)];

crypto/bn/bn_ctx.c:268:1: Parameter `st->depth`
      266.   }
      267.   
      268. > static unsigned int BN_STACK_pop(BN_STACK *st)
      269.   {
      270.       return st->indexes[--(st->depth)];

crypto/bn/bn_ctx.c:268:1: <Length trace>
266.   }
267.   
268. > static unsigned int BN_STACK_pop(BN_STACK *st)
269.   {
270.       return st->indexes[--(st->depth)];

crypto/bn/bn_ctx.c:268:1: Parameter `*st->indexes`
      266.   }
      267.   
      268. > static unsigned int BN_STACK_pop(BN_STACK *st)
      269.   {
      270.       return st->indexes[--(st->depth)];

crypto/bn/bn_ctx.c:270:12: Array access: Offset: [-1, +oo] Size: [1, +oo] by call to `rsa_check_prime_factor_range` 
      268.   static unsigned int BN_STACK_pop(BN_STACK *st)
      269.   {
      270.       return st->indexes[--(st->depth)];
                        ^
      271.   }
      272.   

static unsigned int BN_STACK_pop(BN_STACK *st)
{
    return st->indexes[--(st->depth)];
}
['int rsa_check_prime_factor(BIGNUM *p, BIGNUM *e, int nbits, BN_CTX *ctx)\n{\n    int checks = bn_rsa_fips186_4_prime_MR_min_checks(nbits);\n    int ret = 0;\n    BIGNUM *p1 = NULL, *gcd = NULL;\n    if (BN_is_prime_fasttest_ex(p, checks, ctx, 1, NULL) != 1\n            || rsa_check_prime_factor_range(p, nbits, ctx) != 1)\n        return 0;\n    BN_CTX_start(ctx);\n    p1 = BN_CTX_get(ctx);\n    gcd = BN_CTX_get(ctx);\n    ret = (gcd != NULL)\n          && (BN_copy(p1, p) != NULL)\n          && BN_sub_word(p1, 1)\n          && BN_gcd(gcd, p1, e, ctx)\n          && BN_is_one(gcd);\n    BN_clear(p1);\n    BN_CTX_end(ctx);\n    return ret;\n}', 'int BN_is_prime_fasttest_ex(const BIGNUM *w, int checks, BN_CTX *ctx_passed,\n                            int do_trial_division, BN_GENCB *cb)\n{\n    int i, status, ret = -1;\n    BN_CTX *ctx = NULL;\n    if (BN_cmp(w, BN_value_one()) <= 0)\n        return 0;\n    if (BN_is_odd(w)) {\n        if (BN_is_word(w, 3))\n            return 1;\n    } else {\n        return BN_is_word(w, 2);\n    }\n    if (do_trial_division) {\n        for (i = 1; i < NUMPRIMES; i++) {\n            BN_ULONG mod = BN_mod_word(w, primes[i]);\n            if (mod == (BN_ULONG)-1)\n                return -1;\n            if (mod == 0)\n                return BN_is_word(w, primes[i]);\n        }\n        if (!BN_GENCB_call(cb, 1, -1))\n            return -1;\n    }\n    if (ctx_passed != NULL)\n        ctx = ctx_passed;\n    else if ((ctx = BN_CTX_new()) == NULL)\n        goto err;\n    ret = bn_miller_rabin_is_prime(w, checks, ctx, cb, 0, &status);\n    if (!ret)\n        goto err;\n    ret = (status == BN_PRIMETEST_PROBABLY_PRIME);\nerr:\n    if (ctx_passed == NULL)\n        BN_CTX_free(ctx);\n    return ret;\n}', 'int rsa_check_prime_factor_range(const BIGNUM *p, int nbits, BN_CTX *ctx)\n{\n    int ret = 0;\n    BIGNUM *tmp, *low;\n    nbits >>= 1;\n    if (BN_num_bits(p) != nbits)\n        return 0;\n    BN_CTX_start(ctx);\n    tmp = BN_CTX_get(ctx);\n    low = BN_CTX_get(ctx);\n    if (low == NULL || !BN_set_word(tmp, 0xB504F334))\n        goto err;\n    if (nbits >= 32) {\n        if (!BN_lshift(low, tmp, nbits - 32))\n            goto err;\n    } else if (!BN_rshift(low, tmp, 32 - nbits)) {\n        goto err;\n    }\n    if (BN_cmp(p, low) < 0)\n        goto err;\n    ret = 1;\nerr:\n    BN_CTX_end(ctx);\n    return ret;\n}', 'void BN_CTX_start(BN_CTX *ctx)\n{\n    CTXDBG("ENTER BN_CTX_start()", ctx);\n    if (ctx->err_stack || ctx->too_many)\n        ctx->err_stack++;\n    else if (!BN_STACK_push(&ctx->stack, ctx->used)) {\n        BNerr(BN_F_BN_CTX_START, BN_R_TOO_MANY_TEMPORARY_VARIABLES);\n        ctx->err_stack++;\n    }\n    CTXDBG("LEAVE BN_CTX_start()", ctx);\n}', 'void BN_CTX_end(BN_CTX *ctx)\n{\n    if (ctx == NULL)\n        return;\n    CTXDBG("ENTER BN_CTX_end()", ctx);\n    if (ctx->err_stack)\n        ctx->err_stack--;\n    else {\n        unsigned int fp = BN_STACK_pop(&ctx->stack);\n        if (fp < ctx->used)\n            BN_POOL_release(&ctx->pool, ctx->used - fp);\n        ctx->used = fp;\n        ctx->too_many = 0;\n    }\n    CTXDBG("LEAVE BN_CTX_end()", ctx);\n}', 'static unsigned int BN_STACK_pop(BN_STACK *st)\n{\n    return st->indexes[--(st->depth)];\n}']