test/rsa_sp800_56b_test.c:513: error: INTEGER_OVERFLOW_L2
  ([0, +oo] - 1):unsigned32 by call to `rsa_sp800_56b_derive_params_from_pq`.
Showing all 20 steps of the trace


test/rsa_sp800_56b_test.c:488:5: Call
486.       if (!(TEST_ptr(key = RSA_new()) && TEST_ptr(ctx = BN_CTX_new())))
487.           goto err;
488.       BN_CTX_start(ctx);
           ^
489.   
490.       e = bn_load(ctx, cav_e, sizeof(cav_e));

crypto/bn/bn_ctx.c:171:1: Parameter `ctx->stack.depth`
  169.   }
  170.   
  171. > void BN_CTX_start(BN_CTX *ctx)
  172.   {
  173.       CTXDBG("ENTER BN_CTX_start()", ctx);

test/rsa_sp800_56b_test.c:513:14: Call
511.                                                        Xp2, q1, q2, NULL, Xq, Xq1,
512.                                                        Xq2, 2048, e, ctx, NULL))
513.             && TEST_true(rsa_sp800_56b_derive_params_from_pq(key, 2048, e, ctx))
                    ^
514.             && TEST_BN_eq(p1_exp, p1)
515.             && TEST_BN_eq(p2_exp, p2)

crypto/rsa/rsa_sp800_56b_gen.c:180:1: Parameter `ctx->stack.depth`
  178.    *           1 = success.
  179.    */
  180. > int rsa_sp800_56b_derive_params_from_pq(RSA *rsa, int nbits,
  181.                                           const BIGNUM *e, BN_CTX *ctx)
  182.   {

crypto/rsa/rsa_sp800_56b_gen.c:186:5: Call
  184.       BIGNUM *p1, *q1, *lcm, *p1q1, *gcd;
  185.   
  186.       BN_CTX_start(ctx);
             ^
  187.       p1 = BN_CTX_get(ctx);
  188.       q1 = BN_CTX_get(ctx);

crypto/bn/bn_ctx.c:171:1: Parameter `ctx->stack.depth`
    169.   }
    170.   
    171. > void BN_CTX_start(BN_CTX *ctx)
    172.   {
    173.       CTXDBG("ENTER BN_CTX_start()", ctx);

crypto/rsa/rsa_sp800_56b_gen.c:196:9: Call
  194.   
  195.       /* LCM((p-1, q-1)) */
  196.       if (rsa_get_lcm(ctx, rsa->p, rsa->q, lcm, gcd, p1, q1, p1q1) != 1)
                 ^
  197.           goto err;
  198.   

crypto/rsa/rsa_sp800_56b_check.c:220:1: Parameter `ctx->stack.depth`
    218.   
    219.   /* return LCM(p-1, q-1) */
    220. > int rsa_get_lcm(BN_CTX *ctx, const BIGNUM *p, const BIGNUM *q,
    221.                   BIGNUM *lcm, BIGNUM *gcd, BIGNUM *p1, BIGNUM *q1,
    222.                   BIGNUM *p1q1)

crypto/rsa/rsa_sp800_56b_check.c:226:15: Call
    224.       return BN_sub(p1, p, BN_value_one())    /* p-1 */
    225.              && BN_sub(q1, q, BN_value_one()) /* q-1 */
    226.              && BN_mul(p1q1, p1, q1, ctx)     /* (p-1)(q-1) */
                         ^
    227.              && BN_gcd(gcd, p1, q1, ctx)
    228.              && BN_div(lcm, NULL, p1q1, gcd, ctx); /* LCM((p-1, q-1)) */

crypto/bn/bn_mul.c:497:1: Parameter `ctx->stack.depth`
      495.   #endif                          /* BN_RECURSION */
      496.   
      497. > int BN_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)
      498.   {
      499.       int ret = bn_mul_fixed_top(r, a, b, ctx);

crypto/bn/bn_mul.c:499:15: Call
      497.   int BN_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)
      498.   {
      499.       int ret = bn_mul_fixed_top(r, a, b, ctx);
                           ^
      500.   
      501.       bn_correct_top(r);

crypto/bn/bn_mul.c:507:1: Parameter `ctx->stack.depth`
        505.   }
        506.   
        507. > int bn_mul_fixed_top(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)
        508.   {
        509.       int ret = 0;

crypto/bn/bn_mul.c:533:5: Call
        531.       top = al + bl;
        532.   
        533.       BN_CTX_start(ctx);
                   ^
        534.       if ((r == a) || (r == b)) {
        535.           if ((rr = BN_CTX_get(ctx)) == NULL)

crypto/bn/bn_ctx.c:171:1: Parameter `ctx->stack.depth`
          169.   }
          170.   
          171. > void BN_CTX_start(BN_CTX *ctx)
          172.   {
          173.       CTXDBG("ENTER BN_CTX_start()", ctx);

crypto/bn/bn_mul.c:618:5: Call
        616.    err:
        617.       bn_check_top(r);
        618.       BN_CTX_end(ctx);
                   ^
        619.       return ret;
        620.   }

crypto/bn/bn_ctx.c:185:1: Parameter `ctx->stack.depth`
          183.   }
          184.   
          185. > void BN_CTX_end(BN_CTX *ctx)
          186.   {
          187.       CTXDBG("ENTER BN_CTX_end()", ctx);

crypto/bn/bn_ctx.c:191:27: Call
          189.           ctx->err_stack--;
          190.       else {
          191.           unsigned int fp = BN_STACK_pop(&ctx->stack);
                                           ^
          192.           /* Does this stack frame have anything to release? */
          193.           if (fp < ctx->used)

crypto/bn/bn_ctx.c:266:1: <LHS trace>
264.   }
265.   
266. > static unsigned int BN_STACK_pop(BN_STACK *st)
267.   {
268.       return st->indexes[--(st->depth)];

crypto/bn/bn_ctx.c:266:1: Parameter `st->depth`
            264.   }
            265.   
            266. > static unsigned int BN_STACK_pop(BN_STACK *st)
            267.   {
            268.       return st->indexes[--(st->depth)];

crypto/bn/bn_ctx.c:268:12: Binary operation: ([0, +oo] - 1):unsigned32 by call to `rsa_sp800_56b_derive_params_from_pq` 
            266.   static unsigned int BN_STACK_pop(BN_STACK *st)
            267.   {
            268.       return st->indexes[--(st->depth)];
                              ^
            269.   }
            270.   

static unsigned int BN_STACK_pop(BN_STACK *st)
{
    return st->indexes[--(st->depth)];
}
['static int test_fips1864_keygen_kat(void)\n{\n    int ret = 0;\n    RSA *key = NULL;\n    BN_CTX *ctx = NULL;\n    BIGNUM *e, *Xp, *Xp1, *Xp2, *Xq, *Xq1, *Xq2;\n    BIGNUM *p1, *p2, *q1, *q2;\n    BIGNUM *p1_exp, *p2_exp, *q1_exp, *q2_exp;\n    BIGNUM *p_exp, *q_exp, *n_exp, *d_exp;\n    const BIGNUM *p, *q, *n, *d, *e2;\n    if (!(TEST_ptr(key = RSA_new()) && TEST_ptr(ctx = BN_CTX_new())))\n        goto err;\n    BN_CTX_start(ctx);\n    e = bn_load(ctx, cav_e, sizeof(cav_e));\n    Xp = bn_load(ctx, cav_Xp, sizeof(cav_Xp));\n    Xp1 = bn_load(ctx, cav_Xp1, sizeof(cav_Xp1));\n    Xp2 = bn_load(ctx, cav_Xp2, sizeof(cav_Xp2));\n    Xq = bn_load(ctx, cav_Xq, sizeof(cav_Xq));\n    Xq1 = bn_load(ctx, cav_Xq1, sizeof(cav_Xq1));\n    Xq2 = bn_load(ctx, cav_Xq2, sizeof(cav_Xq2));\n    p1_exp = bn_load(ctx, cav_p1, sizeof(cav_p1));\n    p2_exp = bn_load(ctx, cav_p2, sizeof(cav_p2));\n    q1_exp = bn_load(ctx, cav_q1, sizeof(cav_q1));\n    q2_exp = bn_load(ctx, cav_q2, sizeof(cav_q2));\n    p_exp = bn_load(ctx, cav_p, sizeof(cav_p));\n    q_exp = bn_load(ctx, cav_q, sizeof(cav_q));\n    n_exp = bn_load(ctx, cav_n, sizeof(cav_n));\n    d_exp = bn_load(ctx, cav_d, sizeof(cav_d));\n    p1 = BN_CTX_get(ctx);\n    p2 = BN_CTX_get(ctx);\n    q1 = BN_CTX_get(ctx);\n    q2 = BN_CTX_get(ctx);\n    ret = TEST_ptr(q2)\n          && TEST_true(rsa_fips186_4_gen_prob_primes(key, p1, p2, NULL, Xp, Xp1,\n                                                     Xp2, q1, q2, NULL, Xq, Xq1,\n                                                     Xq2, 2048, e, ctx, NULL))\n          && TEST_true(rsa_sp800_56b_derive_params_from_pq(key, 2048, e, ctx))\n          && TEST_BN_eq(p1_exp, p1)\n          && TEST_BN_eq(p2_exp, p2)\n          && TEST_BN_eq(q1_exp, q1)\n          && TEST_BN_eq(q2_exp, q2);\n    if (!ret)\n        goto err;\n    RSA_get0_key(key, &n, &e2, &d);\n    RSA_get0_factors(key, &p, &q);\n    ret = TEST_BN_eq(e, e2)\n          && TEST_BN_eq(p_exp, p)\n          && TEST_BN_eq(q_exp, q)\n          && TEST_BN_eq(n_exp, n)\n          && TEST_BN_eq(d_exp, d);\nerr:\n    RSA_free(key);\n    BN_CTX_end(ctx);\n    BN_CTX_free(ctx);\n    return ret;\n}', 'void BN_CTX_start(BN_CTX *ctx)\n{\n    CTXDBG("ENTER BN_CTX_start()", ctx);\n    if (ctx->err_stack || ctx->too_many)\n        ctx->err_stack++;\n    else if (!BN_STACK_push(&ctx->stack, ctx->used)) {\n        BNerr(BN_F_BN_CTX_START, BN_R_TOO_MANY_TEMPORARY_VARIABLES);\n        ctx->err_stack++;\n    }\n    CTXDBG("LEAVE BN_CTX_start()", ctx);\n}', 'int rsa_sp800_56b_derive_params_from_pq(RSA *rsa, int nbits,\n                                        const BIGNUM *e, BN_CTX *ctx)\n{\n    int ret = -1;\n    BIGNUM *p1, *q1, *lcm, *p1q1, *gcd;\n    BN_CTX_start(ctx);\n    p1 = BN_CTX_get(ctx);\n    q1 = BN_CTX_get(ctx);\n    lcm = BN_CTX_get(ctx);\n    p1q1 = BN_CTX_get(ctx);\n    gcd = BN_CTX_get(ctx);\n    if (gcd == NULL)\n        goto err;\n    if (rsa_get_lcm(ctx, rsa->p, rsa->q, lcm, gcd, p1, q1, p1q1) != 1)\n        goto err;\n    BN_free(rsa->e);\n    rsa->e = BN_dup(e);\n    if (rsa->e == NULL)\n        goto err;\n    BN_clear_free(rsa->d);\n    rsa->d = BN_secure_new();\n    if (rsa->d == NULL || BN_mod_inverse(rsa->d, e, lcm, ctx) == NULL)\n        goto err;\n    if (BN_num_bits(rsa->d) <= (nbits >> 1)) {\n        ret = 0;\n        goto err;\n    }\n    if (rsa->n == NULL)\n        rsa->n = BN_new();\n    if (rsa->n == NULL || !BN_mul(rsa->n, rsa->p, rsa->q, ctx))\n        goto err;\n    if (rsa->dmp1 == NULL)\n        rsa->dmp1 = BN_new();\n    if (rsa->dmp1 == NULL || !BN_mod(rsa->dmp1, rsa->d, p1, ctx))\n        goto err;\n    if (rsa->dmq1 == NULL)\n        rsa->dmq1 = BN_secure_new();\n    if (rsa->dmq1 == NULL || !BN_mod(rsa->dmq1, rsa->d, q1, ctx))\n        goto err;\n    BN_free(rsa->iqmp);\n    rsa->iqmp = BN_secure_new();\n    if (rsa->iqmp == NULL\n            || BN_mod_inverse(rsa->iqmp, rsa->q, rsa->p, ctx) == NULL)\n        goto err;\n    ret = 1;\nerr:\n    if (ret != 1) {\n        BN_free(rsa->e);\n        rsa->e = NULL;\n        BN_free(rsa->d);\n        rsa->d = NULL;\n        BN_free(rsa->n);\n        rsa->n = NULL;\n        BN_free(rsa->iqmp);\n        rsa->iqmp = NULL;\n        BN_free(rsa->dmq1);\n        rsa->dmq1 = NULL;\n        BN_free(rsa->dmp1);\n        rsa->dmp1 = NULL;\n    }\n    BN_clear(p1);\n    BN_clear(q1);\n    BN_clear(lcm);\n    BN_clear(p1q1);\n    BN_clear(gcd);\n    BN_CTX_end(ctx);\n    return ret;\n}', 'int rsa_get_lcm(BN_CTX *ctx, const BIGNUM *p, const BIGNUM *q,\n                BIGNUM *lcm, BIGNUM *gcd, BIGNUM *p1, BIGNUM *q1,\n                BIGNUM *p1q1)\n{\n    return BN_sub(p1, p, BN_value_one())\n           && BN_sub(q1, q, BN_value_one())\n           && BN_mul(p1q1, p1, q1, ctx)\n           && BN_gcd(gcd, p1, q1, ctx)\n           && BN_div(lcm, NULL, p1q1, gcd, ctx);\n}', 'int BN_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)\n{\n    int ret = bn_mul_fixed_top(r, a, b, ctx);\n    bn_correct_top(r);\n    bn_check_top(r);\n    return ret;\n}', 'int bn_mul_fixed_top(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)\n{\n    int ret = 0;\n    int top, al, bl;\n    BIGNUM *rr;\n#if defined(BN_MUL_COMBA) || defined(BN_RECURSION)\n    int i;\n#endif\n#ifdef BN_RECURSION\n    BIGNUM *t = NULL;\n    int j = 0, k;\n#endif\n    bn_check_top(a);\n    bn_check_top(b);\n    bn_check_top(r);\n    al = a->top;\n    bl = b->top;\n    if ((al == 0) || (bl == 0)) {\n        BN_zero(r);\n        return 1;\n    }\n    top = al + bl;\n    BN_CTX_start(ctx);\n    if ((r == a) || (r == b)) {\n        if ((rr = BN_CTX_get(ctx)) == NULL)\n            goto err;\n    } else\n        rr = r;\n#if defined(BN_MUL_COMBA) || defined(BN_RECURSION)\n    i = al - bl;\n#endif\n#ifdef BN_MUL_COMBA\n    if (i == 0) {\n# if 0\n        if (al == 4) {\n            if (bn_wexpand(rr, 8) == NULL)\n                goto err;\n            rr->top = 8;\n            bn_mul_comba4(rr->d, a->d, b->d);\n            goto end;\n        }\n# endif\n        if (al == 8) {\n            if (bn_wexpand(rr, 16) == NULL)\n                goto err;\n            rr->top = 16;\n            bn_mul_comba8(rr->d, a->d, b->d);\n            goto end;\n        }\n    }\n#endif\n#ifdef BN_RECURSION\n    if ((al >= BN_MULL_SIZE_NORMAL) && (bl >= BN_MULL_SIZE_NORMAL)) {\n        if (i >= -1 && i <= 1) {\n            if (i >= 0) {\n                j = BN_num_bits_word((BN_ULONG)al);\n            }\n            if (i == -1) {\n                j = BN_num_bits_word((BN_ULONG)bl);\n            }\n            j = 1 << (j - 1);\n            assert(j <= al || j <= bl);\n            k = j + j;\n            t = BN_CTX_get(ctx);\n            if (t == NULL)\n                goto err;\n            if (al > j || bl > j) {\n                if (bn_wexpand(t, k * 4) == NULL)\n                    goto err;\n                if (bn_wexpand(rr, k * 4) == NULL)\n                    goto err;\n                bn_mul_part_recursive(rr->d, a->d, b->d,\n                                      j, al - j, bl - j, t->d);\n            } else {\n                if (bn_wexpand(t, k * 2) == NULL)\n                    goto err;\n                if (bn_wexpand(rr, k * 2) == NULL)\n                    goto err;\n                bn_mul_recursive(rr->d, a->d, b->d, j, al - j, bl - j, t->d);\n            }\n            rr->top = top;\n            goto end;\n        }\n    }\n#endif\n    if (bn_wexpand(rr, top) == NULL)\n        goto err;\n    rr->top = top;\n    bn_mul_normal(rr->d, a->d, al, b->d, bl);\n#if defined(BN_MUL_COMBA) || defined(BN_RECURSION)\n end:\n#endif\n    rr->neg = a->neg ^ b->neg;\n    rr->flags |= BN_FLG_FIXED_TOP;\n    if (r != rr && BN_copy(r, rr) == NULL)\n        goto err;\n    ret = 1;\n err:\n    bn_check_top(r);\n    BN_CTX_end(ctx);\n    return ret;\n}', 'void BN_CTX_end(BN_CTX *ctx)\n{\n    CTXDBG("ENTER BN_CTX_end()", ctx);\n    if (ctx->err_stack)\n        ctx->err_stack--;\n    else {\n        unsigned int fp = BN_STACK_pop(&ctx->stack);\n        if (fp < ctx->used)\n            BN_POOL_release(&ctx->pool, ctx->used - fp);\n        ctx->used = fp;\n        ctx->too_many = 0;\n    }\n    CTXDBG("LEAVE BN_CTX_end()", ctx);\n}', 'static unsigned int BN_STACK_pop(BN_STACK *st)\n{\n    return st->indexes[--(st->depth)];\n}']