test/rsa_sp800_56b_test.c:317: error: BUFFER_OVERRUN_L3
  Offset: [-1, +oo] Size: [1, +oo] by call to `rsa_check_private_exponent`.
Showing all 17 steps of the trace


test/rsa_sp800_56b_test.c:317:11: Call
315.       }
316.       /* fails since d >= lcm(p-1, q-1) */
317.       ret = TEST_false(rsa_check_private_exponent(key, 8, ctx))
                 ^
318.             && TEST_true(BN_set_word(d, 45))
319.             /* d is correct size and 1 = e.d mod lcm(p-1, q-1) */

crypto/rsa/rsa_sp800_56b_check.c:157:1: Parameter `ctx->stack.depth`
  155.    *     (Step 6b) 1 = (d*e) mod LCM(p–1, q–1)
  156.    */
  157. > int rsa_check_private_exponent(const RSA *rsa, int nbits, BN_CTX *ctx)
  158.   {
  159.       int ret;

crypto/rsa/rsa_sp800_56b_check.c:166:5: Call
  164.           return 0;
  165.   
  166.       BN_CTX_start(ctx);
             ^
  167.       r = BN_CTX_get(ctx);
  168.       p1 = BN_CTX_get(ctx);

crypto/bn/bn_ctx.c:171:1: Parameter `ctx->stack.depth`
    169.   }
    170.   
    171. > void BN_CTX_start(BN_CTX *ctx)
    172.   {
    173.       CTXDBG("ENTER BN_CTX_start()", ctx);

crypto/rsa/rsa_sp800_56b_check.c:175:15: Call
  173.       ret = (gcd != NULL
  174.             /* LCM(p - 1, q - 1) */
  175.             && (rsa_get_lcm(ctx, rsa->p, rsa->q, lcm, gcd, p1, q1, p1q1) == 1)
                       ^
  176.             /* (Step 6a) d < LCM(p - 1, q - 1) */
  177.             && (BN_cmp(rsa->d, lcm) < 0)

crypto/rsa/rsa_sp800_56b_check.c:220:1: Parameter `ctx->stack.depth`
    218.   
    219.   /* return LCM(p-1, q-1) */
    220. > int rsa_get_lcm(BN_CTX *ctx, const BIGNUM *p, const BIGNUM *q,
    221.                   BIGNUM *lcm, BIGNUM *gcd, BIGNUM *p1, BIGNUM *q1,
    222.                   BIGNUM *p1q1)

crypto/rsa/rsa_sp800_56b_check.c:179:14: Call
  177.             && (BN_cmp(rsa->d, lcm) < 0)
  178.             /* (Step 6b) 1 = (e . d) mod LCM(p - 1, q - 1) */
  179.             && BN_mod_mul(r, rsa->e, rsa->d, lcm, ctx)
                      ^
  180.             && BN_is_one(r));
  181.   

crypto/bn/bn_mod.c:203:5: Call
    201.       bn_check_top(m);
    202.   
    203.       BN_CTX_start(ctx);
               ^
    204.       if ((t = BN_CTX_get(ctx)) == NULL)
    205.           goto err;

crypto/bn/bn_ctx.c:171:1: Parameter `*ctx->stack.indexes`
      169.   }
      170.   
      171. > void BN_CTX_start(BN_CTX *ctx)
      172.   {
      173.       CTXDBG("ENTER BN_CTX_start()", ctx);

crypto/bn/bn_mod.c:218:5: Call
    216.       ret = 1;
    217.    err:
    218.       BN_CTX_end(ctx);
               ^
    219.       return ret;
    220.   }

crypto/bn/bn_ctx.c:185:1: Parameter `*ctx->stack.indexes`
      183.   }
      184.   
      185. > void BN_CTX_end(BN_CTX *ctx)
      186.   {
      187.       if (ctx == NULL)

crypto/bn/bn_ctx.c:193:27: Call
      191.           ctx->err_stack--;
      192.       else {
      193.           unsigned int fp = BN_STACK_pop(&ctx->stack);
                                       ^
      194.           /* Does this stack frame have anything to release? */
      195.           if (fp < ctx->used)

crypto/bn/bn_ctx.c:268:1: <Offset trace>
266.   }
267.   
268. > static unsigned int BN_STACK_pop(BN_STACK *st)
269.   {
270.       return st->indexes[--(st->depth)];

crypto/bn/bn_ctx.c:268:1: Parameter `st->depth`
        266.   }
        267.   
        268. > static unsigned int BN_STACK_pop(BN_STACK *st)
        269.   {
        270.       return st->indexes[--(st->depth)];

crypto/bn/bn_ctx.c:268:1: <Length trace>
266.   }
267.   
268. > static unsigned int BN_STACK_pop(BN_STACK *st)
269.   {
270.       return st->indexes[--(st->depth)];

crypto/bn/bn_ctx.c:268:1: Parameter `*st->indexes`
        266.   }
        267.   
        268. > static unsigned int BN_STACK_pop(BN_STACK *st)
        269.   {
        270.       return st->indexes[--(st->depth)];

crypto/bn/bn_ctx.c:270:12: Array access: Offset: [-1, +oo] Size: [1, +oo] by call to `rsa_check_private_exponent` 
        268.   static unsigned int BN_STACK_pop(BN_STACK *st)
        269.   {
        270.       return st->indexes[--(st->depth)];
                          ^
        271.   }
        272.   

static unsigned int BN_STACK_pop(BN_STACK *st)
{
    return st->indexes[--(st->depth)];
}
['static int test_check_private_exponent(void)\n{\n    int ret = 0;\n    RSA *key = NULL;\n    BN_CTX *ctx = NULL;\n    BIGNUM *p = NULL, *q = NULL, *e = NULL, *d = NULL, *n = NULL;\n    ret = TEST_ptr(key = RSA_new())\n          && TEST_ptr(ctx = BN_CTX_new())\n          && TEST_ptr(p = BN_new())\n          && TEST_ptr(q = BN_new())\n          && TEST_true(BN_set_word(p, 15))\n          && TEST_true(BN_set_word(q, 17))\n          && TEST_true(RSA_set0_factors(key, p, q));\n    if (!ret) {\n        BN_free(p);\n        BN_free(q);\n        goto end;\n    }\n    ret = TEST_ptr(e = BN_new())\n          && TEST_ptr(d = BN_new())\n          && TEST_ptr(n = BN_new())\n          && TEST_true(BN_set_word(e, 5))\n          && TEST_true(BN_set_word(d, 157))\n          && TEST_true(BN_set_word(n, 15*17))\n          && TEST_true(RSA_set0_key(key, n, e, d));\n    if (!ret) {\n        BN_free(e);\n        BN_free(d);\n        BN_free(n);\n        goto end;\n    }\n    ret = TEST_false(rsa_check_private_exponent(key, 8, ctx))\n          && TEST_true(BN_set_word(d, 45))\n          && TEST_true(rsa_check_private_exponent(key, 8, ctx))\n          && TEST_false(rsa_check_private_exponent(key, 16, ctx))\n          && TEST_true(BN_set_word(d, 16))\n          && TEST_false(rsa_check_private_exponent(key, 8, ctx))\n          && TEST_true(BN_set_word(d, 46))\n          && TEST_false(rsa_check_private_exponent(key, 8, ctx));\nend:\n    RSA_free(key);\n    BN_CTX_free(ctx);\n    return ret;\n}', 'int rsa_check_private_exponent(const RSA *rsa, int nbits, BN_CTX *ctx)\n{\n    int ret;\n    BIGNUM *r, *p1, *q1, *lcm, *p1q1, *gcd;\n    if (BN_num_bits(rsa->d) <= (nbits >> 1))\n        return 0;\n    BN_CTX_start(ctx);\n    r = BN_CTX_get(ctx);\n    p1 = BN_CTX_get(ctx);\n    q1 = BN_CTX_get(ctx);\n    lcm = BN_CTX_get(ctx);\n    p1q1 = BN_CTX_get(ctx);\n    gcd = BN_CTX_get(ctx);\n    ret = (gcd != NULL\n          && (rsa_get_lcm(ctx, rsa->p, rsa->q, lcm, gcd, p1, q1, p1q1) == 1)\n          && (BN_cmp(rsa->d, lcm) < 0)\n          && BN_mod_mul(r, rsa->e, rsa->d, lcm, ctx)\n          && BN_is_one(r));\n    BN_clear(p1);\n    BN_clear(q1);\n    BN_clear(lcm);\n    BN_clear(gcd);\n    BN_CTX_end(ctx);\n    return ret;\n}', 'void BN_CTX_start(BN_CTX *ctx)\n{\n    CTXDBG("ENTER BN_CTX_start()", ctx);\n    if (ctx->err_stack || ctx->too_many)\n        ctx->err_stack++;\n    else if (!BN_STACK_push(&ctx->stack, ctx->used)) {\n        BNerr(BN_F_BN_CTX_START, BN_R_TOO_MANY_TEMPORARY_VARIABLES);\n        ctx->err_stack++;\n    }\n    CTXDBG("LEAVE BN_CTX_start()", ctx);\n}', 'int rsa_get_lcm(BN_CTX *ctx, const BIGNUM *p, const BIGNUM *q,\n                BIGNUM *lcm, BIGNUM *gcd, BIGNUM *p1, BIGNUM *q1,\n                BIGNUM *p1q1)\n{\n    return BN_sub(p1, p, BN_value_one())\n           && BN_sub(q1, q, BN_value_one())\n           && BN_mul(p1q1, p1, q1, ctx)\n           && BN_gcd(gcd, p1, q1, ctx)\n           && BN_div(lcm, NULL, p1q1, gcd, ctx);\n}', 'int BN_mod_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m,\n               BN_CTX *ctx)\n{\n    BIGNUM *t;\n    int ret = 0;\n    bn_check_top(a);\n    bn_check_top(b);\n    bn_check_top(m);\n    BN_CTX_start(ctx);\n    if ((t = BN_CTX_get(ctx)) == NULL)\n        goto err;\n    if (a == b) {\n        if (!BN_sqr(t, a, ctx))\n            goto err;\n    } else {\n        if (!BN_mul(t, a, b, ctx))\n            goto err;\n    }\n    if (!BN_nnmod(r, t, m, ctx))\n        goto err;\n    bn_check_top(r);\n    ret = 1;\n err:\n    BN_CTX_end(ctx);\n    return ret;\n}', 'void BN_CTX_end(BN_CTX *ctx)\n{\n    if (ctx == NULL)\n        return;\n    CTXDBG("ENTER BN_CTX_end()", ctx);\n    if (ctx->err_stack)\n        ctx->err_stack--;\n    else {\n        unsigned int fp = BN_STACK_pop(&ctx->stack);\n        if (fp < ctx->used)\n            BN_POOL_release(&ctx->pool, ctx->used - fp);\n        ctx->used = fp;\n        ctx->too_many = 0;\n    }\n    CTXDBG("LEAVE BN_CTX_end()", ctx);\n}', 'static unsigned int BN_STACK_pop(BN_STACK *st)\n{\n    return st->indexes[--(st->depth)];\n}']