title: How to correctly generate a RSA key pair in C?
body: <p>I'm trying to generate a RSA key pair in C with the following function:</p>
  <pre><code>int generate_key(const int bits, char* public_key_name, char* private_key_name){
      int ret = 0;
      RSA *r = NULL;
      BIGNUM *bne = NULL;
      FILE *bp_public = NULL;
      FILE* bp_private = NULL;
      
      unsigned long   e = RSA_F4;
  
      // 1. generate rsa key
      bne = BN_secure_new();
      ret = BN_set_word(bne,e);
      if(ret != 1){
          goto free_all;
      }
  
      r = RSA_new();
      ret = RSA_generate_key_ex(r, bits, bne, NULL);
      if(ret != 1){
          goto free_all;
      }
  
      // 2. save public key
      bp_public = fopen(public_key_name, &quot;w+&quot;);
      ret = PEM_write_RSAPublicKey(bp_public, r);
      fclose(bp_public);
      if(ret != 1){
          goto free_all;
      }
  
      // 3. save private key
      bp_private = fopen(private_key_name, &quot;w+&quot;);
      ret = PEM_write_RSAPrivateKey(bp_private, r, NULL, NULL, 0, NULL, NULL);
      fclose(bp_private);
  
      // 4. free
      free_all:
      RSA_free(r);
      BN_clear_free(bne);
  
      return ret;
  }
  </code></pre>
  <p>It works fine except for the fact that when I used the public key to verify a signed message an error pops out (<code>error:0909006C:PEM routines:get_name:no</code>) as result of <code>EVP_VerifyFinal</code>. Notice that, weird thing, the returning value of <code>EVP_VerifyFinal</code> is not <code>0</code> but <code>-1</code>: if it was <code>0</code> then It would be unrefutably a signature verification failure but it is not, it happens some other thing that I could not figure out. In the following I past the code used to sign and verify.</p>
  <pre><code>void sign(const unsigned char* const restrict clear_buf, const unsigned long clear_size, unsigned char* restrict* const restrict sgnt_buf, unsigned long* const restrict sgnt_size, const char* const restrict prvkey_file_name){
     const EVP_MD* md;
     EVP_MD_CTX* md_ctx;
     int ret;
     FILE* prvkey_file;
     EVP_PKEY* prvkey;
     unsigned expected_sgn_size;
     unsigned tmp;
     
     if(clear_size &gt; INT_MAX){
         fprintf(stderr, &quot;Buffer to sign too big\n&quot;);
         exit(1);
     }
     
     if((prvkey_file = fopen(prvkey_file_name, &quot;r&quot;)) == NULL){
         fprintf(stderr, &quot;Error: cannot open file '%s' (missing?)\n&quot;, prvkey_file_name);
         exit(1);
     }
     prvkey = PEM_read_PrivateKey(prvkey_file, NULL, NULL, NULL);
     fclose(prvkey_file);
     expected_sgn_size = (unsigned) EVP_PKEY_size(prvkey);
  
     if((*sgnt_buf = (unsigned char*)malloc((size_t)expected_sgn_size)) == NULL){
         fprintf(stderr, &quot;Error in allocating memory for signature. Error: %s\n&quot;, strerror(errno));
         exit(1);
     }
     // create the signature context:
     md = EVP_sha256();
     md_ctx = EVP_MD_CTX_new();
     if(!md_ctx){
         fprintf(stderr, &quot;Error: EVP_MD_CTX_new returned NULL\n&quot;);
         exit(1);
     }
  
     if(EVP_SignInit(md_ctx, md) == 0){
         fprintf(stderr, &quot;Error: EVP_SignInit returned %d\n&quot;, ret);
         exit(1);
     }
     if(EVP_SignUpdate(md_ctx, clear_buf, (unsigned)clear_size) == 0){
         fprintf(stderr, &quot;Error: EVP_SignUpdate returned %d\n&quot;, ret);
         exit(1);
     }
     if(EVP_SignFinal(md_ctx, *sgnt_buf, &amp;tmp, prvkey) == 0){
         fprintf(stderr, &quot;Error: EVP_SignFinal returned %d\n&quot;, ret);
         exit(1);
     }
     if(tmp &lt; expected_sgn_size){
         fprintf(stderr, &quot;Error in signing, signature size does not match expected size\n&quot;);
         exit(1);
     }
     
     *sgnt_size = (unsigned long)tmp;
     
     // delete the digest and the private key from memory:
     EVP_MD_CTX_free(md_ctx);
     EVP_PKEY_free(prvkey);
  }
  </code></pre>
  <pre><code>void verify(const unsigned char* const restrict file_buf, unsigned long* const restrict file_size, const char* const restrict pubkey_file_name){
     // declare some useful variables:
     int ret;
     FILE* pubkey_file;
     EVP_PKEY* pubkey;
     unsigned char* sgnt_buf;
     unsigned sgnt_size;
     
     pubkey_file = fopen(pubkey_file_name, &quot;r&quot;);
     if(!pubkey_file){
         fprintf(stderr, &quot;Error: cannot open file '%s' (missing?)\n&quot;, pubkey_file_name);
         exit(1);
     }
     pubkey = PEM_read_PUBKEY(pubkey_file, NULL, NULL, NULL);
     sgnt_size = (unsigned) EVP_PKEY_size(pubkey);
     fclose(pubkey_file);
     
     const EVP_MD* md = EVP_sha256();
     EVP_MD_CTX* md_ctx;
     
     *file_size -= (unsigned long)sgnt_size;
     if((sgnt_buf = (unsigned char*)malloc((size_t)sgnt_size)) == NULL){
         fprintf(stderr, &quot;Error in allocating memory for signature. Error: %s\n&quot;, strerror(errno));
         exit(1);
     }
     memcpy((void*)sgnt_buf, (void*)(file_buf + *file_size), (size_t)sgnt_size);
     
     // create the signature context:
     md_ctx = EVP_MD_CTX_new();
     if(!md_ctx){
         fprintf(stderr, &quot;Error: EVP_MD_CTX_new returned NULL\n&quot;);
         exit(1);
     }
  
     if(EVP_VerifyInit(md_ctx, md) == 0){
         fprintf(stderr, &quot;Error in EVP_VerifyInit\n&quot;);
         exit(1);
     }
     if(EVP_VerifyUpdate(md_ctx, file_buf, *file_size) == 0){
         fprintf(stderr, &quot;Error in EVP_VerifyUpdate\n&quot;);
         exit(1); 
     }
     ret = EVP_VerifyFinal(md_ctx, sgnt_buf, sgnt_size, pubkey);
     if(ret == 0){ // it is 0 if invalid signature, -1 if some other error, 1 if success.
         fprintf(stderr, &quot;Error: EVP_VerifyFinal failed: invalid signature\n&quot;);
         exit(1);
     } else if(ret == -1){
         fprintf(stderr, &quot;Some error occured during signature verification\n&quot;);
         exit(1);
     }else if (ret == 1){
         // fprintf(stdout, &quot;Signature verified\n&quot;);
     }else{
         fprintf(stderr, &quot;I shouldn't be printed. EVP_VerifyFinal returned %d\n&quot;, ret);
         exit(1);
     }
     EVP_MD_CTX_free(md_ctx);
     EVP_PKEY_free(pubkey);
     free(sgnt_buf);
  }
  </code></pre>
  <p>On the web I found some solution but either they are not in C, or/and they are deprecated don't solve the problem (one solution that I found here generates a private key that it doesn't even sign for some reason...)</p>