crypto/rsa/rsa_sp800_56b_check.c:276: error: INTEGER_OVERFLOW_L2
  ([0, +oo] - 1):unsigned32 by call to `BN_gcd`.
Showing all 10 steps of the trace


crypto/rsa/rsa_sp800_56b_check.c:276:10: Call
274.        * The modulus has no factors smaller than 752.
275.        */
276.       if (!BN_gcd(gcd, rsa->n, bn_get0_small_factors(), ctx) || !BN_is_one(gcd)) {
                ^
277.           RSAerr(RSA_F_RSA_SP800_56B_CHECK_PUBLIC, RSA_R_INVALID_MODULUS);
278.           goto err;

crypto/bn/bn_gcd.c:15:1: Parameter `ctx->stack.depth`
  13.   static BIGNUM *euclid(BIGNUM *a, BIGNUM *b);
  14.   
  15. > int BN_gcd(BIGNUM *r, const BIGNUM *in_a, const BIGNUM *in_b, BN_CTX *ctx)
  16.   {
  17.       BIGNUM *a, *b, *t;

crypto/bn/bn_gcd.c:23:5: Call
  21.       bn_check_top(in_b);
  22.   
  23.       BN_CTX_start(ctx);
            ^
  24.       a = BN_CTX_get(ctx);
  25.       b = BN_CTX_get(ctx);

crypto/bn/bn_ctx.c:171:1: Parameter `ctx->stack.depth`
    169.   }
    170.   
    171. > void BN_CTX_start(BN_CTX *ctx)
    172.   {
    173.       CTXDBG("ENTER BN_CTX_start()", ctx);

crypto/bn/bn_gcd.c:49:5: Call
  47.       ret = 1;
  48.    err:
  49.       BN_CTX_end(ctx);
            ^
  50.       bn_check_top(r);
  51.       return ret;

crypto/bn/bn_ctx.c:185:1: Parameter `ctx->stack.depth`
    183.   }
    184.   
    185. > void BN_CTX_end(BN_CTX *ctx)
    186.   {
    187.       CTXDBG("ENTER BN_CTX_end()", ctx);

crypto/bn/bn_ctx.c:191:27: Call
    189.           ctx->err_stack--;
    190.       else {
    191.           unsigned int fp = BN_STACK_pop(&ctx->stack);
                                     ^
    192.           /* Does this stack frame have anything to release? */
    193.           if (fp < ctx->used)

crypto/bn/bn_ctx.c:266:1: <LHS trace>
264.   }
265.   
266. > static unsigned int BN_STACK_pop(BN_STACK *st)
267.   {
268.       return st->indexes[--(st->depth)];

crypto/bn/bn_ctx.c:266:1: Parameter `st->depth`
      264.   }
      265.   
      266. > static unsigned int BN_STACK_pop(BN_STACK *st)
      267.   {
      268.       return st->indexes[--(st->depth)];

crypto/bn/bn_ctx.c:268:12: Binary operation: ([0, +oo] - 1):unsigned32 by call to `BN_gcd` 
      266.   static unsigned int BN_STACK_pop(BN_STACK *st)
      267.   {
      268.       return st->indexes[--(st->depth)];
                        ^
      269.   }
      270.   

static unsigned int BN_STACK_pop(BN_STACK *st)
{
    return st->indexes[--(st->depth)];
}
['int rsa_sp800_56b_check_public(const RSA *rsa)\n{\n    int ret = 0, nbits, iterations, status;\n    BN_CTX *ctx = NULL;\n    BIGNUM *gcd = NULL;\n    if (rsa->n == NULL || rsa->e == NULL)\n        return 0;\n    nbits = BN_num_bits(rsa->n);\n    if (!rsa_sp800_56b_validate_strength(nbits, -1)) {\n        RSAerr(RSA_F_RSA_SP800_56B_CHECK_PUBLIC, RSA_R_INVALID_KEY_LENGTH);\n        return 0;\n    }\n    if (!BN_is_odd(rsa->n)) {\n        RSAerr(RSA_F_RSA_SP800_56B_CHECK_PUBLIC, RSA_R_INVALID_MODULUS);\n        return 0;\n    }\n    if (!rsa_check_public_exponent(rsa->e)) {\n        RSAerr(RSA_F_RSA_SP800_56B_CHECK_PUBLIC,\n               RSA_R_PUB_EXPONENT_OUT_OF_RANGE);\n        return 0;\n    }\n    ctx = BN_CTX_new();\n    gcd = BN_new();\n    if (ctx == NULL || gcd == NULL)\n        goto err;\n    iterations = bn_rsa_fips186_4_prime_MR_min_checks(nbits);\n    if (!BN_gcd(gcd, rsa->n, bn_get0_small_factors(), ctx) || !BN_is_one(gcd)) {\n        RSAerr(RSA_F_RSA_SP800_56B_CHECK_PUBLIC, RSA_R_INVALID_MODULUS);\n        goto err;\n    }\n    ret = bn_miller_rabin_is_prime(rsa->n, iterations, ctx, NULL, 1, &status);\n    if (ret != 1 || status != BN_PRIMETEST_COMPOSITE_NOT_POWER_OF_PRIME) {\n        RSAerr(RSA_F_RSA_SP800_56B_CHECK_PUBLIC, RSA_R_INVALID_MODULUS);\n        ret = 0;\n        goto err;\n    }\n    ret = 1;\nerr:\n    BN_free(gcd);\n    BN_CTX_free(ctx);\n    return ret;\n}', 'int BN_gcd(BIGNUM *r, const BIGNUM *in_a, const BIGNUM *in_b, BN_CTX *ctx)\n{\n    BIGNUM *a, *b, *t;\n    int ret = 0;\n    bn_check_top(in_a);\n    bn_check_top(in_b);\n    BN_CTX_start(ctx);\n    a = BN_CTX_get(ctx);\n    b = BN_CTX_get(ctx);\n    if (b == NULL)\n        goto err;\n    if (BN_copy(a, in_a) == NULL)\n        goto err;\n    if (BN_copy(b, in_b) == NULL)\n        goto err;\n    a->neg = 0;\n    b->neg = 0;\n    if (BN_cmp(a, b) < 0) {\n        t = a;\n        a = b;\n        b = t;\n    }\n    t = euclid(a, b);\n    if (t == NULL)\n        goto err;\n    if (BN_copy(r, t) == NULL)\n        goto err;\n    ret = 1;\n err:\n    BN_CTX_end(ctx);\n    bn_check_top(r);\n    return ret;\n}', 'void BN_CTX_start(BN_CTX *ctx)\n{\n    CTXDBG("ENTER BN_CTX_start()", ctx);\n    if (ctx->err_stack || ctx->too_many)\n        ctx->err_stack++;\n    else if (!BN_STACK_push(&ctx->stack, ctx->used)) {\n        BNerr(BN_F_BN_CTX_START, BN_R_TOO_MANY_TEMPORARY_VARIABLES);\n        ctx->err_stack++;\n    }\n    CTXDBG("LEAVE BN_CTX_start()", ctx);\n}', 'void BN_CTX_end(BN_CTX *ctx)\n{\n    CTXDBG("ENTER BN_CTX_end()", ctx);\n    if (ctx->err_stack)\n        ctx->err_stack--;\n    else {\n        unsigned int fp = BN_STACK_pop(&ctx->stack);\n        if (fp < ctx->used)\n            BN_POOL_release(&ctx->pool, ctx->used - fp);\n        ctx->used = fp;\n        ctx->too_many = 0;\n    }\n    CTXDBG("LEAVE BN_CTX_end()", ctx);\n}', 'static unsigned int BN_STACK_pop(BN_STACK *st)\n{\n    return st->indexes[--(st->depth)];\n}']