title: Issue porting Decryption from Windows CryptoAPI to linux libmcrypt
body: <p>I am trying to port my program from Windows to Linux. The windows program uses Window CryptoAPI and linux is using libmcrypt.</p>
  
  <p>Here is the Windows code:</p>
  
  <pre><code>#include &lt;windows.h&gt;
  #include &lt;iostream&gt;
  #include &lt;iomanip&gt;
  #include &lt;vector&gt;
  #include &lt;string&gt;
  #include &lt;exception&gt;
  
  using namespace std;
  
  class CryptError:public exception{
      public:
          CryptError(){}
  };
  
  
  #define CHECK_RET(x) if(x == FALSE) {retval = GetLastError(); throw CryptError();};
  
  LONG Decrypt(const string &amp;key, std::vector&lt;BYTE&gt; &amp;data){
      LONG retval = 0;
      try{
          HCRYPTPROV hCrypt;
          HCRYPTHASH hHash;
          HCRYPTKEY hKey;
          CHECK_RET(CryptAcquireContext(&amp;hCrypt, NULL, NULL, PROV_RSA_FULL, 0));
          CHECK_RET(CryptCreateHash(hCrypt, CALG_MD5, 0, 0, &amp;hHash));
          CHECK_RET(CryptHashData(hHash, reinterpret_cast&lt;const BYTE *&gt;(key.c_str()), key.size(), 0));
  
          CHECK_RET(CryptDeriveKey(hCrypt, CALG_RC2, hHash, MAKELPARAM(CRYPT_EXPORTABLE, 80), &amp;hKey));
  
          BYTE tempVal[200];
          DWORD len = 200;
          CryptGetKeyParam(hKey, KP_EFFECTIVE_KEYLEN, tempVal, &amp;len, 0);
  
          len = 200;
          CryptGetKeyParam(hKey, KP_MODE, tempVal, &amp;len, 0);
  
          len = 200;
          CryptExportKey(hKey, NULL, PLAINTEXTKEYBLOB, 0, tempVal, &amp;len);
  
          len = 200;
          CryptGetKeyParam(hKey, KP_IV, tempVal, &amp;len, 0);
  
          DWORD count = data.size();
          CHECK_RET(CryptDecrypt(hKey, 0, TRUE, 0, &amp;(data[0]), &amp;count));
          data.resize(count);
      }catch(CryptError &amp;e){
      }
  
      return retval;
  }
  
  int main(void){
      BYTE data[9] = {0xdc,0x3d,0x96,0x23,0x29,0xdd,0x1b,0x2f, 0};
      vector&lt;BYTE&gt; vData(data, data + 8);
  
      Decrypt("PNEMAIL", vData);
  
      cerr &lt;&lt; "vData: ";
      int len = vData.size();
      for(int i = 0; i &lt; len; i++){
          if(i &gt; 0)
              cerr &lt;&lt; ',';
          cerr &lt;&lt; hex &lt;&lt; setw(2) &lt;&lt; setfill('0') &lt;&lt; (int)(vData[i]);
      }
      cerr &lt;&lt; endl;
  
      return 0;
  } 
  </code></pre>
  
  <p>When the program is run, it returns:</p>
  
  <pre><code>vData: 42,46,30,41,43,34,31
  </code></pre>
  
  <p>The Q&amp;D linux version looks like this:</p>
  
  <pre><code>#include &lt;mcrypt.h&gt;
  #include &lt;iostream&gt;
  #include &lt;iomanip&gt;
  #include &lt;string&gt;
  #include &lt;openssl/md5.h&gt;
  #include &lt;stdint.h&gt;
  #include &lt;stdexcept&gt;
  #include &lt;vector&gt;
  #include &lt;valarray&gt;
  #include &lt;memory.h&gt;
  
  using namespace std;
  
  class MCrypt{
      private:
          MCRYPT mcrypt;
      public:
          MCrypt(char *algorithm, char* algorithm_directory, char *mode, char* mode_directory){
              mcrypt = mcrypt_module_open(algorithm, algorithm_directory, mode, mode_directory);
              if(mcrypt == MCRYPT_FAILED)
                  throw runtime_error("MCrypt init failed");
          }
  
          int init(void *key, int lenofkey, void *IV){
              return mcrypt_generic_init(mcrypt, key, lenofkey, IV);
          }
  
          int enc_get_iv_size(){
              return mcrypt_enc_get_iv_size(mcrypt);
          }
  
          int deinit(){
              return mcrypt_generic_deinit(mcrypt);
          }
  
          int decrypt(void *data, int len){
              mdecrypt_generic(mcrypt, data, len);
          }
  
          ~MCrypt(){
              deinit();
              mcrypt_module_close(mcrypt);
          }
  };
  
  #ifdef DEBUG
  void inline printArrayFunc(const char *start, const uint8_t *data, int len){
      // DEBUG: print value of $key1
      cerr &lt;&lt; start;
      for(int i = 0; i &lt; len; i++){
          if(i &gt; 0)
              cerr &lt;&lt; ',';
          cerr &lt;&lt; hex &lt;&lt; setw(2) &lt;&lt; setfill('0') &lt;&lt; (int)(data[i]);
      }
      cerr &lt;&lt; endl;
  }
  #define printArray(start, data, len) printArrayFunc(start, data, len)
  #else
  #define printArray(start, data, len)
  #endif
  
  int main(void){
      uint8_t data[8] = {0xdc,0x3d,0x96,0x23,0x29,0xdd,0x1b,0x2f};
      const char *sKey1 = "PNEMAIL";
      const int key1Len = 7;
  
      uint8_t *dataPtr = &amp;(data[0]);
  
      uint8_t key1[17];
      key1[16] = 0;
  
      // Hash sKey1
      MD5(reinterpret_cast&lt;const unsigned char *&gt;(sKey1), key1Len, key1);
  
      MCrypt mcrypt(MCRYPT_RC2, NULL, MCRYPT_CBC, NULL);
  
      vector&lt;uint8_t&gt; iv(mcrypt.enc_get_iv_size(), 0);
  
      // Use the first 80-bits of key1
      mcrypt.init(key1, 10, &amp;(iv[0]));
  
      mcrypt.decrypt(dataPtr, 8);
  
      printArray("vData: ", dataPtr, 8);
  
      return 0;
  }
  </code></pre>
  
  <p>When the program is run, it returns:</p>
  
  <pre><code>vData: 4d,3d,82,71,88,d2,d5,4b
  </code></pre>
  
  <p>I've check that both programs are using the same data.</p>
  
  <ul>
  <li>CryptDeriveKey creates a key <code>07,f1,e2,ea,d4,c8,79,74,03,a6</code> (according to CryptExportKey), the same as the first 10 bytes of the md5 generated in Linux (which I shorten to match the requested 80-bit key).</li>
  <li>Neither are using a salt on the algorithm (or at least are not reporting as such)</li>
  <li>They are both using an 8-byte IV of {0,0,0,0,0,0,0,0}</li>
  <li>They are both using the RC2 algorithm</li>
  <li>They are both using CBC mode</li>
  </ul>
  
  <p>I cannot figure out why they are returning different data. Any assistance would be greatly appreciated.</p>