engines/e_ubsec.c:594: error: INTEGER_OVERFLOW_L2
  ([0, `p->top`] - 1):unsigned64 by call to `BN_mod_exp`.
Showing all 11 steps of the trace


engines/e_ubsec.c:556:1: Parameter `p->top`
554.   	}
555.   
556. > static int ubsec_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
557.   		const BIGNUM *m, BN_CTX *ctx)
558.   	{

engines/e_ubsec.c:589:3: Call
587.   	if (p_UBSEC_rsa_mod_exp_ioctl(fd, (unsigned char *)a->d, BN_num_bits(a),
588.   		(unsigned char *)m->d, BN_num_bits(m), (unsigned char *)p->d, 
589.   		BN_num_bits(p), (unsigned char *)r->d, &y_len) != 0)
         ^
590.   	{
591.   		UBSECerr(UBSEC_F_UBSEC_MOD_EXP, UBSEC_R_REQUEST_FAILED);

crypto/bn/bn_lib.c:248:1: Parameter `a->top`
  246.   	}
  247.   
  248. > int BN_num_bits(const BIGNUM *a)
  249.   	{
  250.   	int i = a->top - 1;

engines/e_ubsec.c:594:24: Call
592.                   p_UBSEC_ubsec_close(fd);
593.   
594.                   return BN_mod_exp(r, a, p, m, ctx);
                              ^
595.   	}
596.   

crypto/bn/bn_exp.c:163:1: Parameter `p->top`
  161.   
  162.   
  163. > int BN_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, const BIGNUM *m,
  164.   	       BN_CTX *ctx)
  165.   	{

crypto/bn/bn_exp.c:219:8: Call
  217.   			{
  218.   			BN_ULONG A = a->d[0];
  219.   			ret=BN_mod_exp_mont_word(r,A,p,m,ctx,NULL);
                ^
  220.   			}
  221.   		else

crypto/bn/bn_exp.c:727:1: Parameter `p->top`
    725.   	}
    726.   
    727. > int BN_mod_exp_mont_word(BIGNUM *rr, BN_ULONG a, const BIGNUM *p,
    728.                            const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *in_mont)
    729.   	{

crypto/bn/bn_exp.c:769:9: Call
    767.   		a %= m->d[0]; /* make sure that 'a' is reduced */
    768.   
    769.   	bits = BN_num_bits(p);
                   ^
    770.   	if (bits == 0)
    771.   		{

crypto/bn/bn_lib.c:248:1: <LHS trace>
246.   	}
247.   
248. > int BN_num_bits(const BIGNUM *a)
249.   	{
250.   	int i = a->top - 1;

crypto/bn/bn_lib.c:248:1: Parameter `a->top`
      246.   	}
      247.   
      248. > int BN_num_bits(const BIGNUM *a)
      249.   	{
      250.   	int i = a->top - 1;

crypto/bn/bn_lib.c:250:2: Binary operation: ([0, p->top] - 1):unsigned64 by call to `BN_mod_exp` 
      248.   int BN_num_bits(const BIGNUM *a)
      249.   	{
      250.   	int i = a->top - 1;
              ^
      251.   	bn_check_top(a);
      252.   

int BN_num_bits(const BIGNUM *a)
	{
	int i = a->top - 1;
	bn_check_top(a);
	if (BN_is_zero(a)) return 0;
	return ((i*BN_BITS2) + BN_num_bits_word(a->d[i]));
	}
['static int ubsec_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,\n\t\tconst BIGNUM *m, BN_CTX *ctx)\n\t{\n\tint \ty_len = 0;\n\tint \tfd;\n\tif(ubsec_dso == NULL)\n\t{\n\t\tUBSECerr(UBSEC_F_UBSEC_MOD_EXP, UBSEC_R_NOT_LOADED);\n\t\treturn 0;\n\t}\n\ty_len = BN_num_bits(m);\n\tif (y_len > max_key_len) {\n\t\tUBSECerr(UBSEC_F_UBSEC_MOD_EXP, UBSEC_R_SIZE_TOO_LARGE_OR_TOO_SMALL);\n                return BN_mod_exp(r, a, p, m, ctx);\n\t}\n\tif(!bn_wexpand(r, m->top))\n\t{\n\t\tUBSECerr(UBSEC_F_UBSEC_MOD_EXP, UBSEC_R_BN_EXPAND_FAIL);\n\t\treturn 0;\n\t}\n\tif ((fd = p_UBSEC_ubsec_open(UBSEC_KEY_DEVICE_NAME)) <= 0) {\n\t\tfd = 0;\n\t\tUBSECerr(UBSEC_F_UBSEC_MOD_EXP, UBSEC_R_UNIT_FAILURE);\n                return BN_mod_exp(r, a, p, m, ctx);\n\t}\n\tif (p_UBSEC_rsa_mod_exp_ioctl(fd, (unsigned char *)a->d, BN_num_bits(a),\n\t\t(unsigned char *)m->d, BN_num_bits(m), (unsigned char *)p->d,\n\t\tBN_num_bits(p), (unsigned char *)r->d, &y_len) != 0)\n\t{\n\t\tUBSECerr(UBSEC_F_UBSEC_MOD_EXP, UBSEC_R_REQUEST_FAILED);\n                p_UBSEC_ubsec_close(fd);\n                return BN_mod_exp(r, a, p, m, ctx);\n\t}\n\tp_UBSEC_ubsec_close(fd);\n\tr->top = (BN_num_bits(m)+BN_BITS2-1)/BN_BITS2;\n\treturn 1;\n\t}', 'int BN_num_bits(const BIGNUM *a)\n\t{\n\tint i = a->top - 1;\n\tbn_check_top(a);\n\tif (BN_is_zero(a)) return 0;\n\treturn ((i*BN_BITS2) + BN_num_bits_word(a->d[i]));\n\t}', 'int BN_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, const BIGNUM *m,\n\t       BN_CTX *ctx)\n\t{\n\tint ret;\n\tbn_check_top(a);\n\tbn_check_top(p);\n\tbn_check_top(m);\n#define MONT_MUL_MOD\n#define MONT_EXP_WORD\n#define RECP_MUL_MOD\n#ifdef MONT_MUL_MOD\n\tif (BN_is_odd(m))\n\t\t{\n#  ifdef MONT_EXP_WORD\n\t\tif (a->top == 1 && !a->neg && (BN_get_flags(p, BN_FLG_CONSTTIME) == 0))\n\t\t\t{\n\t\t\tBN_ULONG A = a->d[0];\n\t\t\tret=BN_mod_exp_mont_word(r,A,p,m,ctx,NULL);\n\t\t\t}\n\t\telse\n#  endif\n\t\t\tret=BN_mod_exp_mont(r,a,p,m,ctx,NULL);\n\t\t}\n\telse\n#endif\n#ifdef RECP_MUL_MOD\n\t\t{ ret=BN_mod_exp_recp(r,a,p,m,ctx); }\n#else\n\t\t{ ret=BN_mod_exp_simple(r,a,p,m,ctx); }\n#endif\n\tbn_check_top(r);\n\treturn(ret);\n\t}', 'int BN_mod_exp_mont_word(BIGNUM *rr, BN_ULONG a, const BIGNUM *p,\n                         const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *in_mont)\n\t{\n\tBN_MONT_CTX *mont = NULL;\n\tint b, bits, ret=0;\n\tint r_is_one;\n\tBN_ULONG w, next_w;\n\tBIGNUM *d, *r, *t;\n\tBIGNUM *swap_tmp;\n#define BN_MOD_MUL_WORD(r, w, m) \\\n\t\t(BN_mul_word(r, (w)) && \\\n\t\t(   \\\n\t\t\t(BN_mod(t, r, m, ctx) && (swap_tmp = r, r = t, t = swap_tmp, 1))))\n#define BN_TO_MONTGOMERY_WORD(r, w, mont) \\\n\t\t(BN_set_word(r, (w)) && BN_to_montgomery(r, r, (mont), ctx))\n\tif (BN_get_flags(p, BN_FLG_CONSTTIME) != 0)\n\t\t{\n\t\tBNerr(BN_F_BN_MOD_EXP_MONT_WORD,ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED);\n\t\treturn -1;\n\t\t}\n\tbn_check_top(p);\n\tbn_check_top(m);\n\tif (!BN_is_odd(m))\n\t\t{\n\t\tBNerr(BN_F_BN_MOD_EXP_MONT_WORD,BN_R_CALLED_WITH_EVEN_MODULUS);\n\t\treturn(0);\n\t\t}\n\tif (m->top == 1)\n\t\ta %= m->d[0];\n\tbits = BN_num_bits(p);\n\tif (bits == 0)\n\t\t{\n\t\tret = BN_one(rr);\n\t\treturn ret;\n\t\t}\n\tif (a == 0)\n\t\t{\n\t\tBN_zero(rr);\n\t\tret = 1;\n\t\treturn ret;\n\t\t}\n\tBN_CTX_start(ctx);\n\td = BN_CTX_get(ctx);\n\tr = BN_CTX_get(ctx);\n\tt = BN_CTX_get(ctx);\n\tif (d == NULL || r == NULL || t == NULL) goto err;\n\tif (in_mont != NULL)\n\t\tmont=in_mont;\n\telse\n\t\t{\n\t\tif ((mont = BN_MONT_CTX_new()) == NULL) goto err;\n\t\tif (!BN_MONT_CTX_set(mont, m, ctx)) goto err;\n\t\t}\n\tr_is_one = 1;\n\tw = a;\n\tfor (b = bits-2; b >= 0; b--)\n\t\t{\n\t\tnext_w = w*w;\n\t\tif ((next_w/w) != w)\n\t\t\t{\n\t\t\tif (r_is_one)\n\t\t\t\t{\n\t\t\t\tif (!BN_TO_MONTGOMERY_WORD(r, w, mont)) goto err;\n\t\t\t\tr_is_one = 0;\n\t\t\t\t}\n\t\t\telse\n\t\t\t\t{\n\t\t\t\tif (!BN_MOD_MUL_WORD(r, w, m)) goto err;\n\t\t\t\t}\n\t\t\tnext_w = 1;\n\t\t\t}\n\t\tw = next_w;\n\t\tif (!r_is_one)\n\t\t\t{\n\t\t\tif (!BN_mod_mul_montgomery(r, r, r, mont, ctx)) goto err;\n\t\t\t}\n\t\tif (BN_is_bit_set(p, b))\n\t\t\t{\n\t\t\tnext_w = w*a;\n\t\t\tif ((next_w/a) != w)\n\t\t\t\t{\n\t\t\t\tif (r_is_one)\n\t\t\t\t\t{\n\t\t\t\t\tif (!BN_TO_MONTGOMERY_WORD(r, w, mont)) goto err;\n\t\t\t\t\tr_is_one = 0;\n\t\t\t\t\t}\n\t\t\t\telse\n\t\t\t\t\t{\n\t\t\t\t\tif (!BN_MOD_MUL_WORD(r, w, m)) goto err;\n\t\t\t\t\t}\n\t\t\t\tnext_w = a;\n\t\t\t\t}\n\t\t\tw = next_w;\n\t\t\t}\n\t\t}\n\tif (w != 1)\n\t\t{\n\t\tif (r_is_one)\n\t\t\t{\n\t\t\tif (!BN_TO_MONTGOMERY_WORD(r, w, mont)) goto err;\n\t\t\tr_is_one = 0;\n\t\t\t}\n\t\telse\n\t\t\t{\n\t\t\tif (!BN_MOD_MUL_WORD(r, w, m)) goto err;\n\t\t\t}\n\t\t}\n\tif (r_is_one)\n\t\t{\n\t\tif (!BN_one(rr)) goto err;\n\t\t}\n\telse\n\t\t{\n\t\tif (!BN_from_montgomery(rr, r, mont, ctx)) goto err;\n\t\t}\n\tret = 1;\nerr:\n\tif ((in_mont == NULL) && (mont != NULL)) BN_MONT_CTX_free(mont);\n\tBN_CTX_end(ctx);\n\tbn_check_top(rr);\n\treturn(ret);\n\t}']