<commit_before>// main.cc
//
// Copyright (c) 2016 Frank Lin (lin.xiaoe.f@gmail.com)
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// Expect command is : ./<this_cmd> --input <folder_name> --output <folder_name> --band <band> --maxzerror <max_z_error>

#include <sys/types.h>
#include <sys/stat.h>
#include <dirent.h>
#include <unistd.h>
#include <stdio.h>
#include <string.h>

#include <string>

#include "lerc_util.h"

void create_directory(const char* directory) {
  struct stat st = {0};
  if (stat(directory, &st) == -1) {
    mkdir(directory, 0700);
  }
}

const char *get_filename_ext(const char *filename) {
  const char *dot = strrchr(filename, '.');
  if(!dot || dot == filename) return "";
  return dot + 1;
}

void list_files_do_stuff(const char* name, int level, const std::string& input_path,
                         const std::string& output_path, double max_z_error, int band,
                         bool signed_type) {
  gago::Logger::LogD("DEBUG - working %s", name);
  gago::Logger::LogD("DT_DIR is %d, DT_UNKOWN is %d", DT_DIR, DT_UNKNOWN);

  DIR *dir;
  struct dirent *entry;

  if (!(dir = opendir(name)))
    return;
  if (!(entry = readdir(dir)))
    return;

  do {
    gago::Logger::LogD("!!! - %s is %d", entry->d_name, entry->d_type);

    if (entry->d_type == DT_DIR) {
      gago::Logger::LogD("DEBUG - step2/ DIR - working %s", entry->d_name);

      char path[1024];
      int len = snprintf(path, sizeof(path) - 1, "%s/%s", name, entry->d_name);
      path[len] = 0;
      if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
        continue;

      // create directory in dest folder.
      std::string spec_output_folder = name;
      spec_output_folder += "/";
      spec_output_folder += entry->d_name;
      spec_output_folder.replace(spec_output_folder.begin(), spec_output_folder.begin() + input_path.size(), output_path);
      create_directory(spec_output_folder.c_str());

      // continue
      list_files_do_stuff(path, level + 1, input_path, output_path, max_z_error, band, signed_type);
    } else {
      gago::Logger::LogD("DEBUG - step2/ FILE - working %s", entry->d_name);

      if ((0 == strcmp("tif", get_filename_ext(entry->d_name))) ||
          (0 == strcmp("tiff", get_filename_ext(entry->d_name)))) { // allow tif and tiff extension
        // destination path
        std::string spec_output_folder = name;
        spec_output_folder += "/";
        spec_output_folder += entry->d_name;
        std::string file_path = spec_output_folder;

        gago::Logger::LogD("DEBUG - step3/ FILE - working %s", file_path.c_str());

        size_t lastindex = spec_output_folder.find_last_of(".");
        std::string dest_file_name = spec_output_folder.substr(0, lastindex);
        dest_file_name += ".lerc";
        dest_file_name.replace(dest_file_name.begin(), dest_file_name.begin() + input_path.size(), output_path);

        bool success = gago::LercUtil::EncodeTiffOrDie(file_path,
                                                      dest_file_name,
                                                      max_z_error,
                                                      gago::LercUtil::LercVersion::V2_3,
                                                      gago::LercUtil::DataType::UNKNOWN,
                                                      band,
                                                      signed_type);
        if (!success) {
          gago::Logger::LogD("%s encode failed", file_path.c_str());
        }
      }
    }
  } while ((entry = readdir(dir)));
  closedir(dir);
}

int main(int argc, const char * argv[]) {
  std::string input_folder_path;
  std::string output_folder_path;
  uint32_t band = 0;
  double max_z_error = 0; // losses
  bool signed_type = false;

  // parse input arguments
  for (int i = 0; i < argc; ++i) {
    if (0 == strcmp("--input", argv[i])) {
      input_folder_path = argv[i + 1];
    } else if (0 == strcmp("--output", argv[i])) {
      output_folder_path = argv[i + 1];
    } else if (0 == strcmp("--band", argv[i])) {
      band = atoi(argv[i + 1]);
    } else if (0 == strcmp("--maxzerror", argv[i])) {
      max_z_error = atof(argv[i + 1]);
    } else if (0 == strcmp("--signed", argv[i])) {
      if (0 == strcmp("true", argv[i])) {
        signed_type = true;
      }
    }
  }

  // give a galance
  gago::Logger::LogD("The input folder path is %s, output lerc files will be inside %s, the TIFF band is %d, max Z error given is %f",
                     input_folder_path.c_str(),
                     output_folder_path.c_str(),
                     band,
                     max_z_error);

  // create output directory
  create_directory(output_folder_path.c_str());

  // enumerate all files in directory
  list_files_do_stuff(input_folder_path.c_str(),
                      0,
                      input_folder_path,
                      output_folder_path,
                      max_z_error,
                      band,
                      signed_type);

  gago::Logger::LogD("DONE");

  return EXIT_SUCCESS;
}
<commit_msg>Use stat replacing dir.<commit_after>// main.cc
//
// Copyright (c) 2016 Frank Lin (lin.xiaoe.f@gmail.com)
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// Expect command is : ./<this_cmd> --input <folder_name> --output <folder_name> --band <band> --maxzerror <max_z_error>

#include <sys/types.h>
#include <sys/stat.h>
#include <dirent.h>
#include <unistd.h>
#include <stdio.h>
#include <string.h>

#include <string>

#include "lerc_util.h"

void create_directory(const char* directory) {
  struct stat st = {0};
  if (stat(directory, &st) == -1) {
    mkdir(directory, 0700);
  }
}

const char *get_filename_ext(const char *filename) {
  const char *dot = strrchr(filename, '.');
  if(!dot || dot == filename) return "";
  return dot + 1;
}

void list_files_do_stuff(const char* name, int level, const std::string& input_path,
                         const std::string& output_path, double max_z_error, int band,
                         bool signed_type) {
  gago::Logger::LogD("DEBUG - working %s", name);
  gago::Logger::LogD("DT_DIR is %d, DT_UNKOWN is %d", DT_DIR, DT_UNKNOWN);

  DIR *dir;
  struct dirent *entry;

  if (!(dir = opendir(name)))
    return;
  if (!(entry = readdir(dir)))
    return;

  do {
    bool isDir = false;
    if (entry->d_type == DT_UNKNOWN) {
      struct stat st;
      if (stat(name, &st) == 0) {
        isDir = S_ISDIR(st.st_mode);
      }
    } else if (entry->d_type == DT_DIR) {
      isDir = true;
    }

    if (isDir) {
      gago::Logger::LogD("DEBUG - step2/ DIR - working %s", entry->d_name);
      char path[1024];
      int len = snprintf(path, sizeof(path) - 1, "%s/%s", name, entry->d_name);
      path[len] = 0;
      if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
        continue;

      // create directory in dest folder.
      std::string spec_output_folder = name;
      spec_output_folder += "/";
      spec_output_folder += entry->d_name;
      spec_output_folder.replace(spec_output_folder.begin(), spec_output_folder.begin() + input_path.size(), output_path);
      create_directory(spec_output_folder.c_str());

      // continue
      list_files_do_stuff(path, level + 1, input_path, output_path, max_z_error, band, signed_type);
    } else {
      gago::Logger::LogD("DEBUG - step2/ FILE - working %s", entry->d_name);

      if ((0 == strcmp("tif", get_filename_ext(entry->d_name))) ||
          (0 == strcmp("tiff", get_filename_ext(entry->d_name)))) { // allow tif and tiff extension
        // destination path
        std::string spec_output_folder = name;
        spec_output_folder += "/";
        spec_output_folder += entry->d_name;
        std::string file_path = spec_output_folder;

        gago::Logger::LogD("DEBUG - step3/ FILE - working %s", file_path.c_str());

        size_t lastindex = spec_output_folder.find_last_of(".");
        std::string dest_file_name = spec_output_folder.substr(0, lastindex);
        dest_file_name += ".lerc";
        dest_file_name.replace(dest_file_name.begin(), dest_file_name.begin() + input_path.size(), output_path);

        bool success = gago::LercUtil::EncodeTiffOrDie(file_path,
                                                      dest_file_name,
                                                      max_z_error,
                                                      gago::LercUtil::LercVersion::V2_3,
                                                      gago::LercUtil::DataType::UNKNOWN,
                                                      band,
                                                      signed_type);
        if (!success) {
          gago::Logger::LogD("%s encode failed", file_path.c_str());
        }
      }
    }
  } while ((entry = readdir(dir)));
  closedir(dir);
}

int main(int argc, const char * argv[]) {
  std::string input_folder_path;
  std::string output_folder_path;
  uint32_t band = 0;
  double max_z_error = 0; // losses
  bool signed_type = false;

  // parse input arguments
  for (int i = 0; i < argc; ++i) {
    if (0 == strcmp("--input", argv[i])) {
      input_folder_path = argv[i + 1];
    } else if (0 == strcmp("--output", argv[i])) {
      output_folder_path = argv[i + 1];
    } else if (0 == strcmp("--band", argv[i])) {
      band = atoi(argv[i + 1]);
    } else if (0 == strcmp("--maxzerror", argv[i])) {
      max_z_error = atof(argv[i + 1]);
    } else if (0 == strcmp("--signed", argv[i])) {
      if (0 == strcmp("true", argv[i])) {
        signed_type = true;
      }
    }
  }

  // give a galance
  gago::Logger::LogD("The input folder path is %s, output lerc files will be inside %s, the TIFF band is %d, max Z error given is %f",
                     input_folder_path.c_str(),
                     output_folder_path.c_str(),
                     band,
                     max_z_error);

  // create output directory
  create_directory(output_folder_path.c_str());

  // enumerate all files in directory
  list_files_do_stuff(input_folder_path.c_str(),
                      0,
                      input_folder_path,
                      output_folder_path,
                      max_z_error,
                      band,
                      signed_type);

  gago::Logger::LogD("DONE");

  return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>/*
* Copyright (c) 2013-2020, The PurpleI2P Project
*
* This file is part of Purple i2pd project and licensed under BSD3
*
* See full license text in LICENSE file at top of project tree
*/

#include <stdlib.h>
#include <string.h>

#include "Base.h"

namespace i2p
{
namespace data
{
	static const char T32[32] =
	{
		'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h',
		'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p',
		'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
		'y', 'z', '2', '3', '4', '5', '6', '7',
	};

	const char * GetBase32SubstitutionTable ()
	{
		return T32;
	}

	static void iT64Build(void);

	/*
	*
	* BASE64 Substitution Table
	* -------------------------
	*
	* Direct Substitution Table
	*/

	static const char T64[64] =
	{
		'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
		'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
		'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
		'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
		'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
		'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
		'w', 'x', 'y', 'z', '0', '1', '2', '3',
		'4', '5', '6', '7', '8', '9', '-', '~'
	};

	const char * GetBase64SubstitutionTable ()
	{
		return T64;
	}

	/*
	* Reverse Substitution Table (built in run time)
	*/

	static char iT64[256];
	static int isFirstTime = 1;

	/*
	* Padding
	*/

	static char P64 = '=';

	/*
	*
	* ByteStreamToBase64
	* ------------------
	*
	* Converts binary encoded data to BASE64 format.
	*
	*/

	size_t ByteStreamToBase64 (   /* Number of bytes in the encoded buffer */
		const uint8_t * InBuffer, /* Input buffer, binary data */
		size_t InCount,           /* Number of bytes in the input buffer */
		char * OutBuffer,         /* output buffer */
		size_t len                /* length of output buffer */
	)
	{
		unsigned char * ps;
		unsigned char * pd;
		unsigned char   acc_1;
		unsigned char   acc_2;
		int             i;
		int             n;
		int             m;
		size_t          outCount;

		ps = (unsigned char *)InBuffer;
		n = InCount / 3;
		m = InCount % 3;
		if (!m)
			outCount = 4 * n;
		else
			outCount = 4 * (n + 1);

		if (outCount > len) return 0;

		pd = (unsigned char *)OutBuffer;
		for ( i = 0; i < n; i++ )
		{
			acc_1 = *ps++;
			acc_2 = (acc_1 << 4) & 0x30;
			acc_1 >>= 2;                 /* base64 digit #1 */
			*pd++ = T64[acc_1];
			acc_1 = *ps++;
			acc_2 |= acc_1 >> 4;         /* base64 digit #2 */
			*pd++ = T64[acc_2];
			acc_1 &= 0x0f;
			acc_1 <<= 2;
			acc_2 = *ps++;
			acc_1 |= acc_2 >> 6;         /* base64 digit #3 */
			*pd++ = T64[acc_1];
			acc_2 &= 0x3f;               /* base64 digit #4 */
			*pd++ = T64[acc_2];
		}
		if ( m == 1 )
		{
			acc_1 = *ps++;
			acc_2 = (acc_1 << 4) & 0x3f; /* base64 digit #2 */
			acc_1 >>= 2;                 /* base64 digit #1 */
			*pd++ = T64[acc_1];
			*pd++ = T64[acc_2];
			*pd++ = P64;
			*pd++ = P64;

		}
		else if ( m == 2 )
		{
			acc_1 = *ps++;
			acc_2 = (acc_1 << 4) & 0x3f;
			acc_1 >>= 2;                 /* base64 digit #1 */
			*pd++ = T64[acc_1];
			acc_1 = *ps++;
			acc_2 |= acc_1 >> 4;         /* base64 digit #2 */
			*pd++ = T64[acc_2];
			acc_1 &= 0x0f;
			acc_1 <<= 2;                 /* base64 digit #3 */
			*pd++ = T64[acc_1];
			*pd++ = P64;
		}

		return outCount;
	}

	/*
	*
	* Base64ToByteStream
	* ------------------
	*
	* Converts BASE64 encoded data to binary format. If input buffer is
	* not properly padded, buffer of negative length is returned
	*
	*/

	size_t Base64ToByteStream ( /* Number of output bytes */
		const char * InBuffer,  /* BASE64 encoded buffer */
		size_t InCount,         /* Number of input bytes */
		uint8_t * OutBuffer,    /* output buffer length */
		size_t len              /* length of output buffer */
	)
	{
		unsigned char * ps;
		unsigned char * pd;
		unsigned char   acc_1;
		unsigned char   acc_2;
		int             i;
		int             n;
		int             m;
		size_t          outCount;

		if (isFirstTime)
			iT64Build();

		n = InCount / 4;
		m = InCount % 4;

		if (InCount && !m)
			outCount = 3 * n;
		else
		{
			outCount = 0;
			return 0;
		}

		ps = (unsigned char *)(InBuffer + InCount - 1);
		while ( *ps-- == P64 )
			outCount--;
		ps = (unsigned char *)InBuffer;

		if (outCount > len)
			return -1;

		pd = OutBuffer;
		auto endOfOutBuffer = OutBuffer + outCount;
		for ( i = 0; i < n; i++ )
		{
			acc_1 = iT64[*ps++];
			acc_2 = iT64[*ps++];
			acc_1 <<= 2;
			acc_1 |= acc_2 >> 4;
			*pd++ = acc_1;
			if (pd >= endOfOutBuffer)
				break;

			acc_2 <<= 4;
			acc_1 = iT64[*ps++];
			acc_2 |= acc_1 >> 2;
			*pd++ = acc_2;
			if (pd >= endOfOutBuffer)
				break;

			acc_2 = iT64[*ps++];
			acc_2 |= acc_1 << 6;
			*pd++ = acc_2;
		}

		return outCount;
	}

	size_t Base64EncodingBufferSize (const size_t input_size)
	{
		auto d = div (input_size, 3);
		if (d.rem)
			d.quot++;

		return 4 * d.quot;
	}

	std::string ToBase64Standard (const std::string& in)
	{
		auto len = Base64EncodingBufferSize (in.length ());
		char * str = new char[len + 1];
		auto l = ByteStreamToBase64 ((const uint8_t *)in.c_str (), in.length (), str, len);
		str[l] = 0;
		// replace '-' by '+' and '~' by '/'
		for (size_t i = 0; i < l; i++)
			if (str[i] == '-')
				str[i] = '+';
			else if (str[i] == '~')
				str[i] = '/';

		std::string s(str);
		delete[] str;
		return s;
	}

	/*
	*
	* iT64
	* ----
	* Reverse table builder. P64 character is replaced with 0
	*
	*
	*/

	static void iT64Build()
	{
		int i;
		isFirstTime = 0;
		for ( i = 0; i < 256; i++ ) iT64[i] = -1;
		for ( i = 0; i < 64; i++ ) iT64[(int)T64[i]] = i;
		iT64[(int)P64] = 0;
	}

	size_t Base32ToByteStream (const char * inBuf, size_t len, uint8_t * outBuf, size_t outLen)
	{
		int tmp = 0, bits = 0;
		size_t ret = 0;
		for (size_t i = 0; i < len; i++)
		{
			char ch = inBuf[i];
			if (ch >= '2' && ch <= '7') // digit
				ch = (ch - '2') + 26; // 26 means a-z
			else if (ch >= 'a' && ch <= 'z')
				ch = ch - 'a'; // a = 0
			else
				return 0; // unexpected character

			tmp |= ch;
			bits += 5;
			if (bits >= 8)
			{
				if (ret >= outLen) return ret;
				outBuf[ret] = tmp >> (bits - 8);
				bits -= 8;
				ret++;
			}
			tmp <<= 5;
		}
		return ret;
	}

	size_t ByteStreamToBase32 (const uint8_t * inBuf, size_t len, char * outBuf, size_t outLen)
	{
		size_t ret = 0, pos = 1;
		int bits = 8, tmp = inBuf[0];
		while (ret < outLen && (bits > 0 || pos < len))
		{
			if (bits < 5)
			{
				if (pos < len)
				{
					tmp <<= 8;
					tmp |= inBuf[pos] & 0xFF;
					pos++;
					bits += 8;
				}
				else // last byte
				{
					tmp <<= (5 - bits);
					bits = 5;
				}
			}

			bits -= 5;
			int ind = (tmp >> bits) & 0x1F;
			outBuf[ret] = (ind < 26) ? (ind + 'a') : ((ind - 26) + '2');
			ret++;
		}
		return ret;
	}
}
}
<commit_msg>fixed incorrect ret code<commit_after>/*
* Copyright (c) 2013-2022, The PurpleI2P Project
*
* This file is part of Purple i2pd project and licensed under BSD3
*
* See full license text in LICENSE file at top of project tree
*/

#include <stdlib.h>
#include <string.h>

#include "Base.h"

namespace i2p
{
namespace data
{
	static const char T32[32] =
	{
		'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h',
		'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p',
		'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
		'y', 'z', '2', '3', '4', '5', '6', '7',
	};

	const char * GetBase32SubstitutionTable ()
	{
		return T32;
	}

	static void iT64Build(void);

	/*
	*
	* BASE64 Substitution Table
	* -------------------------
	*
	* Direct Substitution Table
	*/

	static const char T64[64] =
	{
		'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
		'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
		'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
		'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
		'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
		'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
		'w', 'x', 'y', 'z', '0', '1', '2', '3',
		'4', '5', '6', '7', '8', '9', '-', '~'
	};

	const char * GetBase64SubstitutionTable ()
	{
		return T64;
	}

	/*
	* Reverse Substitution Table (built in run time)
	*/

	static char iT64[256];
	static int isFirstTime = 1;

	/*
	* Padding
	*/

	static char P64 = '=';

	/*
	*
	* ByteStreamToBase64
	* ------------------
	*
	* Converts binary encoded data to BASE64 format.
	*
	*/

	size_t ByteStreamToBase64 (   /* Number of bytes in the encoded buffer */
		const uint8_t * InBuffer, /* Input buffer, binary data */
		size_t InCount,           /* Number of bytes in the input buffer */
		char * OutBuffer,         /* output buffer */
		size_t len                /* length of output buffer */
	)
	{
		unsigned char * ps;
		unsigned char * pd;
		unsigned char   acc_1;
		unsigned char   acc_2;
		int             i;
		int             n;
		int             m;
		size_t          outCount;

		ps = (unsigned char *)InBuffer;
		n = InCount / 3;
		m = InCount % 3;
		if (!m)
			outCount = 4 * n;
		else
			outCount = 4 * (n + 1);

		if (outCount > len) return 0;

		pd = (unsigned char *)OutBuffer;
		for ( i = 0; i < n; i++ )
		{
			acc_1 = *ps++;
			acc_2 = (acc_1 << 4) & 0x30;
			acc_1 >>= 2;                 /* base64 digit #1 */
			*pd++ = T64[acc_1];
			acc_1 = *ps++;
			acc_2 |= acc_1 >> 4;         /* base64 digit #2 */
			*pd++ = T64[acc_2];
			acc_1 &= 0x0f;
			acc_1 <<= 2;
			acc_2 = *ps++;
			acc_1 |= acc_2 >> 6;         /* base64 digit #3 */
			*pd++ = T64[acc_1];
			acc_2 &= 0x3f;               /* base64 digit #4 */
			*pd++ = T64[acc_2];
		}
		if ( m == 1 )
		{
			acc_1 = *ps++;
			acc_2 = (acc_1 << 4) & 0x3f; /* base64 digit #2 */
			acc_1 >>= 2;                 /* base64 digit #1 */
			*pd++ = T64[acc_1];
			*pd++ = T64[acc_2];
			*pd++ = P64;
			*pd++ = P64;

		}
		else if ( m == 2 )
		{
			acc_1 = *ps++;
			acc_2 = (acc_1 << 4) & 0x3f;
			acc_1 >>= 2;                 /* base64 digit #1 */
			*pd++ = T64[acc_1];
			acc_1 = *ps++;
			acc_2 |= acc_1 >> 4;         /* base64 digit #2 */
			*pd++ = T64[acc_2];
			acc_1 &= 0x0f;
			acc_1 <<= 2;                 /* base64 digit #3 */
			*pd++ = T64[acc_1];
			*pd++ = P64;
		}

		return outCount;
	}

	/*
	*
	* Base64ToByteStream
	* ------------------
	*
	* Converts BASE64 encoded data to binary format. If input buffer is
	* not properly padded, buffer of negative length is returned
	*
	*/

	size_t Base64ToByteStream ( /* Number of output bytes */
		const char * InBuffer,  /* BASE64 encoded buffer */
		size_t InCount,         /* Number of input bytes */
		uint8_t * OutBuffer,    /* output buffer length */
		size_t len              /* length of output buffer */
	)
	{
		unsigned char * ps;
		unsigned char * pd;
		unsigned char   acc_1;
		unsigned char   acc_2;
		int             i;
		int             n;
		int             m;
		size_t          outCount;

		if (isFirstTime)
			iT64Build();

		n = InCount / 4;
		m = InCount % 4;

		if (InCount && !m)
			outCount = 3 * n;
		else
			return 0;

		ps = (unsigned char *)(InBuffer + InCount - 1);
		while ( *ps-- == P64 )
			outCount--;
		ps = (unsigned char *)InBuffer;

		if (outCount > len)
			return 0;

		pd = OutBuffer;
		auto endOfOutBuffer = OutBuffer + outCount;
		for ( i = 0; i < n; i++ )
		{
			acc_1 = iT64[*ps++];
			acc_2 = iT64[*ps++];
			acc_1 <<= 2;
			acc_1 |= acc_2 >> 4;
			*pd++ = acc_1;
			if (pd >= endOfOutBuffer)
				break;

			acc_2 <<= 4;
			acc_1 = iT64[*ps++];
			acc_2 |= acc_1 >> 2;
			*pd++ = acc_2;
			if (pd >= endOfOutBuffer)
				break;

			acc_2 = iT64[*ps++];
			acc_2 |= acc_1 << 6;
			*pd++ = acc_2;
		}

		return outCount;
	}

	size_t Base64EncodingBufferSize (const size_t input_size)
	{
		auto d = div (input_size, 3);
		if (d.rem)
			d.quot++;

		return 4 * d.quot;
	}

	std::string ToBase64Standard (const std::string& in)
	{
		auto len = Base64EncodingBufferSize (in.length ());
		char * str = new char[len + 1];
		auto l = ByteStreamToBase64 ((const uint8_t *)in.c_str (), in.length (), str, len);
		str[l] = 0;
		// replace '-' by '+' and '~' by '/'
		for (size_t i = 0; i < l; i++)
			if (str[i] == '-')
				str[i] = '+';
			else if (str[i] == '~')
				str[i] = '/';

		std::string s(str);
		delete[] str;
		return s;
	}

	/*
	*
	* iT64
	* ----
	* Reverse table builder. P64 character is replaced with 0
	*
	*
	*/

	static void iT64Build()
	{
		int i;
		isFirstTime = 0;
		for ( i = 0; i < 256; i++ ) iT64[i] = -1;
		for ( i = 0; i < 64; i++ ) iT64[(int)T64[i]] = i;
		iT64[(int)P64] = 0;
	}

	size_t Base32ToByteStream (const char * inBuf, size_t len, uint8_t * outBuf, size_t outLen)
	{
		int tmp = 0, bits = 0;
		size_t ret = 0;
		for (size_t i = 0; i < len; i++)
		{
			char ch = inBuf[i];
			if (ch >= '2' && ch <= '7') // digit
				ch = (ch - '2') + 26; // 26 means a-z
			else if (ch >= 'a' && ch <= 'z')
				ch = ch - 'a'; // a = 0
			else
				return 0; // unexpected character

			tmp |= ch;
			bits += 5;
			if (bits >= 8)
			{
				if (ret >= outLen) return ret;
				outBuf[ret] = tmp >> (bits - 8);
				bits -= 8;
				ret++;
			}
			tmp <<= 5;
		}
		return ret;
	}

	size_t ByteStreamToBase32 (const uint8_t * inBuf, size_t len, char * outBuf, size_t outLen)
	{
		size_t ret = 0, pos = 1;
		int bits = 8, tmp = inBuf[0];
		while (ret < outLen && (bits > 0 || pos < len))
		{
			if (bits < 5)
			{
				if (pos < len)
				{
					tmp <<= 8;
					tmp |= inBuf[pos] & 0xFF;
					pos++;
					bits += 8;
				}
				else // last byte
				{
					tmp <<= (5 - bits);
					bits = 5;
				}
			}

			bits -= 5;
			int ind = (tmp >> bits) & 0x1F;
			outBuf[ret] = (ind < 26) ? (ind + 'a') : ((ind - 26) + '2');
			ret++;
		}
		return ret;
	}
}
}
<|endoftext|>


<commit_before>/*
 * Copyright 2007-2021 CM4all GmbH
 * All rights reserved.
 *
 * author: Max Kellermann <mk@cm4all.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the
 * distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "istream/UringIstream.hxx"
#include "istream/sink_fd.hxx"
#include "istream/UnusedPtr.hxx"
#include "system/Error.hxx"
#include "system/KernelVersion.hxx"
#include "io/Open.hxx"
#include "io/SpliceSupport.hxx"
#include "event/uring/Manager.hxx"
#include "io/uring/Handler.hxx"
#include "io/uring/OpenStat.hxx"
#include "util/PrintException.hxx"
#include "memory/fb_pool.hxx"
#include "PInstance.hxx"

#include <sys/types.h>
#include <sys/stat.h>
#include <stdio.h>
#include <stdlib.h>

struct Context final : PInstance, Uring::OpenStatHandler, SinkFdHandler {
	Uring::Manager uring_manager;
	Uring::OpenStat open_stat;

	const char *_path;

	SinkFd *sink = nullptr;
	std::exception_ptr error;

	Context()
		:uring_manager(event_loop),
		 open_stat(uring_manager, *this) {}

	void BeginShutdown() noexcept {
		uring_manager.SetVolatile();
	}

	void Open(const char *path) noexcept;

	void CreateSinkFd(const char *path,
			  UniqueFileDescriptor &&fd,
			  off_t size) noexcept;

	/* virtual methods from class Uring::OpenStatHandler */
	void OnOpenStat(UniqueFileDescriptor fd,
			struct statx &st) noexcept override;
	void OnOpenStatError(std::exception_ptr e) noexcept override;

	/* virtual methods from class SinkFdHandler */
	void OnInputEof() noexcept override;
	void OnInputError(std::exception_ptr ep) noexcept override;
	bool OnSendError(int error) noexcept override;
};

inline void
Context::Open(const char *path) noexcept
{
	_path = path;
	open_stat.StartOpenStatReadOnly(path);
}

void
Context::CreateSinkFd(const char *path, UniqueFileDescriptor &&fd,
		      off_t size) noexcept
{
	sink = sink_fd_new(event_loop, root_pool,
			   NewUringIstream(uring_manager,
					   root_pool, path,
					   std::move(fd),
					   0, size),
			   FileDescriptor(STDOUT_FILENO),
			   guess_fd_type(STDOUT_FILENO),
			   *this);
}

void
Context::OnOpenStat(UniqueFileDescriptor fd,
		    struct statx &st) noexcept
try {
	if (!S_ISREG(st.stx_mode))
		throw std::runtime_error("Not a regular file");

	CreateSinkFd(_path, std::move(fd), st.stx_size);
} catch (...) {
	error = std::current_exception();
	BeginShutdown();
}

void
Context::OnOpenStatError(std::exception_ptr e) noexcept
{
	error = std::move(e);
	BeginShutdown();
}

void
Context::OnInputEof() noexcept
{
	sink = nullptr;
	BeginShutdown();
}

void
Context::OnInputError(std::exception_ptr ep) noexcept
{
	sink = nullptr;
	error = std::move(ep);
	BeginShutdown();
}

bool
Context::OnSendError(int _error) noexcept
{
	sink = nullptr;
	error = std::make_exception_ptr(MakeErrno(_error, "Failed to write"));
	BeginShutdown();

	return true;
}

int
main(int argc, char **argv)
try {
	if (argc != 2) {
		fprintf(stderr, "Usage: %s PATH\n", argv[0]);
		return EXIT_FAILURE;
	}

	const char *path = argv[1];

	direct_global_init();

	const ScopeFbPoolInit fb_pool_init;

	Context context;
	context.Open(path);

	context.event_loop.Dispatch();

	if (context.error)
		std::rethrow_exception(context.error);

	return EXIT_SUCCESS;
} catch (...) {
	PrintException(std::current_exception());
	return EXIT_FAILURE;
}
<commit_msg>test/RunUringIstream: include cleanup<commit_after>/*
 * Copyright 2007-2021 CM4all GmbH
 * All rights reserved.
 *
 * author: Max Kellermann <mk@cm4all.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the
 * distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "istream/UringIstream.hxx"
#include "istream/sink_fd.hxx"
#include "istream/UnusedPtr.hxx"
#include "system/Error.hxx"
#include "io/Open.hxx"
#include "io/SpliceSupport.hxx"
#include "event/uring/Manager.hxx"
#include "io/uring/Handler.hxx"
#include "io/uring/OpenStat.hxx"
#include "util/PrintException.hxx"
#include "memory/fb_pool.hxx"
#include "PInstance.hxx"

#include <sys/types.h>
#include <sys/stat.h>
#include <stdio.h>
#include <stdlib.h>

struct Context final : PInstance, Uring::OpenStatHandler, SinkFdHandler {
	Uring::Manager uring_manager;
	Uring::OpenStat open_stat;

	const char *_path;

	SinkFd *sink = nullptr;
	std::exception_ptr error;

	Context()
		:uring_manager(event_loop),
		 open_stat(uring_manager, *this) {}

	void BeginShutdown() noexcept {
		uring_manager.SetVolatile();
	}

	void Open(const char *path) noexcept;

	void CreateSinkFd(const char *path,
			  UniqueFileDescriptor &&fd,
			  off_t size) noexcept;

	/* virtual methods from class Uring::OpenStatHandler */
	void OnOpenStat(UniqueFileDescriptor fd,
			struct statx &st) noexcept override;
	void OnOpenStatError(std::exception_ptr e) noexcept override;

	/* virtual methods from class SinkFdHandler */
	void OnInputEof() noexcept override;
	void OnInputError(std::exception_ptr ep) noexcept override;
	bool OnSendError(int error) noexcept override;
};

inline void
Context::Open(const char *path) noexcept
{
	_path = path;
	open_stat.StartOpenStatReadOnly(path);
}

void
Context::CreateSinkFd(const char *path, UniqueFileDescriptor &&fd,
		      off_t size) noexcept
{
	sink = sink_fd_new(event_loop, root_pool,
			   NewUringIstream(uring_manager,
					   root_pool, path,
					   std::move(fd),
					   0, size),
			   FileDescriptor(STDOUT_FILENO),
			   guess_fd_type(STDOUT_FILENO),
			   *this);
}

void
Context::OnOpenStat(UniqueFileDescriptor fd,
		    struct statx &st) noexcept
try {
	if (!S_ISREG(st.stx_mode))
		throw std::runtime_error("Not a regular file");

	CreateSinkFd(_path, std::move(fd), st.stx_size);
} catch (...) {
	error = std::current_exception();
	BeginShutdown();
}

void
Context::OnOpenStatError(std::exception_ptr e) noexcept
{
	error = std::move(e);
	BeginShutdown();
}

void
Context::OnInputEof() noexcept
{
	sink = nullptr;
	BeginShutdown();
}

void
Context::OnInputError(std::exception_ptr ep) noexcept
{
	sink = nullptr;
	error = std::move(ep);
	BeginShutdown();
}

bool
Context::OnSendError(int _error) noexcept
{
	sink = nullptr;
	error = std::make_exception_ptr(MakeErrno(_error, "Failed to write"));
	BeginShutdown();

	return true;
}

int
main(int argc, char **argv)
try {
	if (argc != 2) {
		fprintf(stderr, "Usage: %s PATH\n", argv[0]);
		return EXIT_FAILURE;
	}

	const char *path = argv[1];

	direct_global_init();

	const ScopeFbPoolInit fb_pool_init;

	Context context;
	context.Open(path);

	context.event_loop.Dispatch();

	if (context.error)
		std::rethrow_exception(context.error);

	return EXIT_SUCCESS;
} catch (...) {
	PrintException(std::current_exception());
	return EXIT_FAILURE;
}
<|endoftext|>


<commit_before>// RUN: clang -fsyntax-only -verify %s
class A { };

class B1 : A { };

class B2 : virtual A { };

class B3 : virtual virtual A { }; // expected-error{{duplicate 'virtual' in base specifier}}

class C : public B1, private B2 { };

class D;

class E : public D { }; // expected-error{{base class has incomplete type}}

typedef int I;

class F : public I { }; // expected-error{{base specifier must name a class}}

union U1 : public A { }; // expected-error{{unions cannot have base classes}}

union U2 {};

class G : public U2 { }; // expected-error{{unions cannot be base classes}}

typedef G G_copy;
typedef G G_copy_2;
typedef G_copy G_copy_3;

class H : G_copy, A, G_copy_2, // expected-error{{base class 'G_copy' specified more than once as a direct base class}}
          public G_copy_3 { }; // expected-error{{base class 'G_copy' specified more than once as a direct base class}}
<commit_msg>Add expected note.  Surely people test before the check in stuff.<commit_after>// RUN: clang -fsyntax-only -verify %s
class A { };

class B1 : A { };

class B2 : virtual A { };

class B3 : virtual virtual A { }; // expected-error{{duplicate 'virtual' in base specifier}}

class C : public B1, private B2 { };

class D;		// expected-note {{forward declaration of 'class D'}}

class E : public D { }; // expected-error{{base class has incomplete type}}

typedef int I;

class F : public I { }; // expected-error{{base specifier must name a class}}

union U1 : public A { }; // expected-error{{unions cannot have base classes}}

union U2 {};

class G : public U2 { }; // expected-error{{unions cannot be base classes}}

typedef G G_copy;
typedef G G_copy_2;
typedef G_copy G_copy_3;

class H : G_copy, A, G_copy_2, // expected-error{{base class 'G_copy' specified more than once as a direct base class}}
          public G_copy_3 { }; // expected-error{{base class 'G_copy' specified more than once as a direct base class}}
<|endoftext|>


<commit_before>/*
 * SimpleSimClient.hpp
 *
 *  Created on: 19.04.2016
 *      Author: Marc Hartung
 */

#ifndef TEST_SIMPLESIMCLIENT_HPP_
#define TEST_SIMPLESIMCLIENT_HPP_

#include <cmath>
#include <iostream>

#include <SimulationClient.hpp>

int clientFunc(const std::string & hostname, int port) {
	NetOff::SimulationClient noFC(hostname, port);

	// test if server can be reached
	if (!noFC.initializeConnection())
		throw std::runtime_error("Couldn't reach important server.");

	// send server data about the sim u want to calculate and get a handle of the sim
	int funSim = noFC.addSimulation("/home/of/very/funny/Sim.sim");

	// checkout the variables of the sim
    NetOff::VariableList allInputs = noFC.getPossibleInputVariableNames(funSim);
	NetOff::VariableList allOutputs = noFC.getPossibleOuputVariableNames(funSim);

	// set the input variables to send (in this case just the 3rd,4th variable of all sim variables)
	NetOff::VariableList inputVars;
	inputVars.addReal(allInputs.getReals()[0]); // add the 1st variable
	inputVars.addReal(allInputs.getReals()[1]); // add the 2d variable
	double inputValue[] = { 0.0, 0.0 }; // for all inputs added inputs, the client needs to set initial values

	// set the output variables, which should be received (in this case all sim variables)
	NetOff::VariableList outputVars;
	outputVars = allOutputs;

	// initialize the added simulation:
	noFC.initializeSimulation(funSim, inputVars, outputVars, inputValue,
			nullptr, nullptr);
	noFC.getSimulationFile(funSim,"./test_src.txt","./test_target.txt");

	// no its possible to access the containers of the initialized simulation
	NetOff::ValueContainer & data = noFC.getOutputValueContainer(funSim);

	// start the server, only possible if all added simulations are initialized
	if (!noFC.start())
		throw std::runtime_error("Couldn't start important server.");
	double t = 0;

	while (t < 10.0) {

		if (std::abs(t - 5.0) < 0.01) {
			// pause for fun
			noFC.pause();
			//Do something else or just unpause again
			noFC.unpause();
		}

		NetOff::ValueContainer & inputs = noFC.getInputValueContainer(funSim);
		// set inputs
		inputs.getRealValues()[0] = std::sin(t);
		inputs.getRealValues()[1] = std::cos(t);
		std::cout << "inputs: " << inputs << "\n";

		// after start() its possible to use the communication functions
		// send inputs
		noFC.sendInputValues(funSim, t, inputs);

		// receive outputs
		NetOff::ValueContainer & outputs = noFC.recvOutputValues(funSim, t); // Don't worry. The won't be any data allocated. And the reference '&' is a must. It's not allowed to be copied

		// do something with outputs
		std::cout << "At time " << t << ": " << outputs << "\n";

		t += 0.1;
	}
	// stop the connection:
	noFC.deinitialize();

	return 0;
}

#endif /* TEST_SIMPLESIMCLIENT_HPP_ */
<commit_msg>Correct documentation.<commit_after>/*
 * SimpleSimClient.hpp
 *
 *  Created on: 19.04.2016
 *      Author: Marc Hartung
 */

#ifndef TEST_SIMPLESIMCLIENT_HPP_
#define TEST_SIMPLESIMCLIENT_HPP_

#include <cmath>
#include <iostream>

#include <SimulationClient.hpp>

int clientFunc(const std::string & hostname, int port) {
	NetOff::SimulationClient noFC(hostname, port);

	// test if server can be reached
	if (!noFC.initializeConnection())
		throw std::runtime_error("Couldn't reach important server.");

	// send server data about the sim you want to calculate and get a handle of the sim
	int funSim = noFC.addSimulation("/home/of/very/funny/Sim.sim");

	// checkout the variables of the sim
    NetOff::VariableList allInputs = noFC.getPossibleInputVariableNames(funSim);
	NetOff::VariableList allOutputs = noFC.getPossibleOuputVariableNames(funSim);

	// set the input variables to send (in this case just the 1st and 2nd variable of all sim variables)
	NetOff::VariableList inputVars;
	inputVars.addReal(allInputs.getReals()[0]); // add the 1st variable
	inputVars.addReal(allInputs.getReals()[1]); // add the 2d variable
	double inputValue[] = { 0.0, 0.0 }; // for all inputs added inputs, the client needs to set initial values

	// set the output variables, which should be received (in this case all sim variables)
	NetOff::VariableList outputVars;
	outputVars = allOutputs;

	// initialize the added simulation:
	noFC.initializeSimulation(funSim, inputVars, outputVars, inputValue,
			nullptr, nullptr);
	noFC.getSimulationFile(funSim,"./test_src.txt","./test_target.txt");

	// now its possible to access the containers of the initialized simulation
	NetOff::ValueContainer & data = noFC.getOutputValueContainer(funSim);

	// start the server, only possible if all added simulations are initialized
	if (!noFC.start())
		throw std::runtime_error("Couldn't start important server.");
	double t = 0;

	while (t < 10.0) {

		if (std::abs(t - 5.0) < 0.01) {
			// pause for fun
			noFC.pause();
			// do something else or just unpause again
			noFC.unpause();
		}

		NetOff::ValueContainer & inputs = noFC.getInputValueContainer(funSim);
		// set inputs
		inputs.getRealValues()[0] = std::sin(t);
		inputs.getRealValues()[1] = std::cos(t);
		std::cout << "inputs: " << inputs << "\n";

		// after start() its possible to use the communication functions
		// send inputs
		noFC.sendInputValues(funSim, t, inputs);

		// receive outputs
		NetOff::ValueContainer & outputs = noFC.recvOutputValues(funSim, t); // Don't worry. The won't be any data allocated. And the reference '&' is a must. It's not allowed to be copied

		// do something with outputs
		std::cout << "At time " << t << ": " << outputs << "\n";

		t += 0.1;
	}
	// stop the connection:
	noFC.deinitialize();

	return 0;
}

#endif /* TEST_SIMPLESIMCLIENT_HPP_ */
<|endoftext|>


<commit_before>/**
 * @file sparse_coding.cpp
 * @author Nishant Mehta
 *
 * Implementation of Sparse Coding with Dictionary Learning using l1 (LASSO) or
 * l1+l2 (Elastic Net) regularization.
 */
#include "sparse_coding.hpp"

using namespace std;
using namespace arma;
using namespace mlpack;
using namespace mlpack::regression;
using namespace mlpack::sparse_coding;

// TODO: parameterizable; options to methods?
#define OBJ_TOL 1e-2 // 1E-9
#define NEWTON_TOL 1e-6 // 1E-9

SparseCoding::SparseCoding(const mat& data,
                           const size_t atoms,
                           const double lambda1,
                           const double lambda2) :
    atoms(atoms),
    data(data),
    codes(mat(atoms, data.n_cols)),
    lambda1(lambda1),
    lambda2(lambda2)
{ /* Nothing left to do. */ }

// Always a not good decision!
void SparseCoding::RandomInitDictionary() {
  dictionary = randn(data.n_rows, atoms);

  for (size_t j = 0; j < atoms; ++j)
    dictionary.col(j) /= norm(dictionary.col(j), 2);
}

// The sensible heuristic.
void SparseCoding::DataDependentRandomInitDictionary()
{
  dictionary = mat(data.n_rows, atoms);
  for (size_t j = 0; j < atoms; ++j)
  {
    vec vecD_j = dictionary.unsafe_col(j);
    RandomAtom(vecD_j);
  }
}

void SparseCoding::RandomAtom(vec& atom)
{
  atom.zeros();
  const size_t nSeedAtoms = 3;
  for (size_t i = 0; i < nSeedAtoms; i++)
    atom += data.col(rand() % data.n_cols);

  atom /= norm(atom, 2);
}

void SparseCoding::DoSparseCoding(const size_t maxIterations)
{
  double lastObjVal = DBL_MAX;

  Log::Info << "Initial Coding Step." << endl;

  OptimizeCode();
  uvec adjacencies = find(codes);

  Log::Info << "  Sparsity level: "
      << 100.0 * ((double) (adjacencies.n_elem)) / ((double)
      (atoms * data.n_cols)) << "%" << endl;
  Log::Info << "  Objective value: " << Objective() << "." << endl;

  for (size_t t = 1; t != maxIterations; ++t)
  {
    Log::Info << "Iteration " << t << " of " << maxIterations << "." << endl;

    Log::Info << "Performing dictionary step... ";
    OptimizeDictionary(adjacencies);
    Log::Info << "objective value: " << Objective() << "." << endl;

    Log::Info << "Performing coding step..." << endl;
    OptimizeCode();
    adjacencies = find(codes);
    Log::Info << "  Sparsity level: "
        << 100.0 *
        ((double) (adjacencies.n_elem)) / ((double) (atoms * data.n_cols))
        << "%" << endl;

    double curObjVal = Objective();
    Log::Info << "  Objective value: " << curObjVal << "." << endl;

    double objValImprov = lastObjVal - curObjVal;
    Log::Info << "  Improvement: " << scientific << objValImprov << "." << endl;

    if (objValImprov < OBJ_TOL)
    {
      Log::Info << "Converged within tolerance " << OBJ_TOL << ".\n";
      break;
    }

    lastObjVal = curObjVal;
  }
}

void SparseCoding::OptimizeCode()
{
  // When using Cholesky version of LARS, this is correct even if lambda2 > 0.
  mat matGram = trans(dictionary) * dictionary;
  // mat matGram;
  // if(lambda2 > 0) {
  //   matGram = trans(dictionary) * dictionary + lambda2 * eye(atoms, atoms);
  // }
  // else {
  //   matGram = trans(dictionary) * dictionary;
  // }

  for (size_t i = 0; i < data.n_cols; ++i)
  {
    // Report progress.
    if ((i % 100) == 0)
      Log::Debug << "Optimization at point " << i << "." << endl;

    bool useCholesky = true;
    LARS lars(useCholesky, matGram, lambda1, lambda2);

    //lars->SetGramMem(matGram.memptr(), matGram.n_rows);
    lars.DoLARS(dictionary, data.unsafe_col(i));

    vec beta;
    lars.Solution(beta);
    codes.col(i) = beta;
  }
}

void SparseCoding::OptimizeDictionary(const uvec& adjacencies)
{
  // Count the number of atomic neighbors for each point x^i.
  uvec neighborCounts = zeros<uvec>(data.n_cols, 1);

  if (adjacencies.n_elem > 0)
  {
    // This gets the column index.
    // TODO: is this integer division intentional?
    size_t curPointInd = (size_t) (adjacencies(0) / atoms);
    size_t curCount = 1;

    for (size_t l = 1; l < adjacencies.n_elem; ++l)
    {
      if ((size_t) (adjacencies(l) / atoms) == curPointInd)
      {
        ++curCount;
      }
      else
      {
        neighborCounts(curPointInd) = curCount;
        curPointInd = (size_t) (adjacencies(l) / atoms);
        curCount = 1;
      }
    }

    neighborCounts(curPointInd) = curCount;
  }

  // Handle the case of inactive atoms (atoms not used in the given coding).
  std::vector<size_t> inactiveAtoms;
  std::vector<size_t> activeAtoms;
  activeAtoms.reserve(atoms);

  for (size_t j = 0; j < atoms; ++j)
  {
    if (accu(codes.row(j) != 0) == 0)
      inactiveAtoms.push_back(j);
    else
      activeAtoms.push_back(j);
  }

  const size_t nActiveAtoms = activeAtoms.size();
  const size_t nInactiveAtoms = inactiveAtoms.size();

  // Efficient construction of Z restricted to active atoms.
  mat matActiveZ;
  if (inactiveAtoms.empty())
  {
    matActiveZ = codes;
  }
  else
  {
    uvec inactiveAtomsVec = conv_to<uvec>::from(inactiveAtoms);
    RemoveRows(codes, inactiveAtomsVec, matActiveZ);
  }

  uvec atomReverseLookup = uvec(atoms);
  for (size_t i = 0; i < nActiveAtoms; ++i)
    atomReverseLookup(activeAtoms[i]) = i;

  if (nInactiveAtoms > 0)
  {
    Log::Info << "There are " << nInactiveAtoms
        << " inactive atoms. They will be re-initialized randomly.\n";
  }

  Log::Debug << "Solving Dual via Newton's Method.\n";

  mat dictionaryEstimate;
  // Solve using Newton's method in the dual - note that the final dot
  // multiplication with inv(A) seems to be unavoidable. Although more
  // expensive, the code written this way (we use solve()) should be more
  // numerically stable than just using inv(A) for everything.
  vec dualVars = zeros<vec>(nActiveAtoms);

  //vec dualVars = 1e-14 * ones<vec>(nActiveAtoms);

  // Method used by feature sign code - fails miserably here.  Perhaps the
  // MATLAB optimizer fmincon does something clever?
  //vec dualVars = 10.0 * randu(nActiveAtoms, 1);

  //vec dualVars = diagvec(solve(dictionary, data * trans(codes))
  //    - codes * trans(codes));
  //for (size_t i = 0; i < dualVars.n_elem; i++)
  //  if (dualVars(i) < 0)
  //    dualVars(i) = 0;

  bool converged = false;
  mat codesXT = matActiveZ * trans(data);
  mat codesZT = matActiveZ * trans(matActiveZ);

  for (size_t t = 1; !converged; ++t)
  {
    mat A = codesZT + diagmat(dualVars);

    mat matAInvZXT = solve(A, codesXT);

    vec gradient = -(sum(square(matAInvZXT), 1) - ones<vec>(nActiveAtoms));

    mat hessian = -(-2 * (matAInvZXT * trans(matAInvZXT)) % inv(A));

    vec searchDirection = -solve(hessian, gradient);
    //vec searchDirection = -gradient;

    // Armijo line search.
    const double c = 1e-4;
    double alpha = 1.0;
    const double rho = 0.9;
    double sufficientDecrease = c * dot(gradient, searchDirection);

    /*
    {
      double sumDualVars = sum(dualVars);
      double fOld = -(-trace(trans(codesXT) * matAInvZXT) - sumDualVars);
      Log::Debug << "fOld = " << fOld << "." << endl;
      double fNew =
          -(-trace(trans(codesXT) * solve(codesZT +
          diagmat(dualVars + alpha * searchDirection), codesXT))
          - (sumDualVars + alpha * sum(searchDirection)) );
      Log::Debug << "fNew = " << fNew << "." << endl;
    }
    */

    double improvement;
    while (true)
    {
      // Calculate objective.
      double sumDualVars = sum(dualVars);
      double fOld = -(-trace(trans(codesXT) * matAInvZXT) - sumDualVars);
      double fNew = -(-trace(trans(codesXT) * solve(codesZT +
          diagmat(dualVars + alpha * searchDirection), codesXT)) -
          (sumDualVars + alpha * sum(searchDirection)));

      if (fNew <= fOld + alpha * sufficientDecrease)
      {
        searchDirection = alpha * searchDirection;
        improvement = fOld - fNew;
        break;
      }

      alpha *= rho;
    }

    // End of Armijo line search code.

    dualVars += searchDirection;
    double normGradient = norm(gradient, 2);
    Log::Debug << "Newton Method iteration " << t << ":" << endl;
    Log::Debug << "  Gradient norm: " << std::scientific << normGradient
        << "." << endl;
    Log::Debug << "  Improvement: " << std::scientific << improvement << ".\n";

    if (improvement < NEWTON_TOL)
      converged = true;
  }

  if (inactiveAtoms.empty())
  {
    dictionaryEstimate = trans(solve(codesZT + diagmat(dualVars), codesXT));
  }
  else
  {
    mat dictionaryActiveEstimate = trans(solve(codesZT + diagmat(dualVars),
        codesXT));
    dictionaryEstimate = zeros(data.n_rows, atoms);

    for (size_t i = 0; i < nActiveAtoms; ++i)
      dictionaryEstimate.col(activeAtoms[i]) = dictionaryActiveEstimate.col(i);

    for (size_t i = 0; i < nInactiveAtoms; ++i)
    {
      vec vecdictionaryi = dictionaryEstimate.unsafe_col(inactiveAtoms[i]);
      RandomAtom(vecdictionaryi);
    }
  }

  dictionary = dictionaryEstimate;
}

void SparseCoding::ProjectDictionary()
{
  for (size_t j = 0; j < atoms; j++)
  {
    double normD_j = norm(dictionary.col(j), 2);
    if ((normD_j > 1) && (normD_j - 1.0 > 1e-9))
    {
      Log::Warn << "Norm exceeded 1 by " << std::scientific << normD_j - 1.0
          << ".  Shrinking...\n";
      dictionary.col(j) /= normD_j;
    }
  }
}

double SparseCoding::Objective()
{
  double l11NormZ = sum(sum(abs(codes)));
  double froNormResidual = norm(data - dictionary * codes, "fro");

  if (lambda2 > 0)
  {
    double froNormZ = norm(codes, "fro");
    return 0.5 *
      (froNormResidual * froNormResidual + lambda2 * froNormZ * froNormZ) +
      lambda1 * l11NormZ;
  }
  else
  {
    return 0.5 * froNormResidual * froNormResidual + lambda1 * l11NormZ;
  }
}

void mlpack::sparse_coding::RemoveRows(const mat& X,
                                       uvec rows_to_remove,
                                       mat& X_mod)
{
  const size_t n_cols = X.n_cols;
  const size_t n_rows = X.n_rows;
  const size_t n_to_remove = rows_to_remove.n_elem;
  const size_t n_to_keep = n_rows - n_to_remove;

  if (n_to_remove == 0)
  {
    X_mod = X;
  }
  else
  {
    X_mod.set_size(n_to_keep, n_cols);

    size_t cur_row = 0;
    size_t remove_ind = 0;
    // First, check 0 to first row to remove.
    if (rows_to_remove(0) > 0)
    {
      // Note that this implies that n_rows > 1.
      size_t height = rows_to_remove(0);
      X_mod(span(cur_row, cur_row + height - 1), span::all) =
          X(span(0, rows_to_remove(0) - 1), span::all);
      cur_row += height;
    }
    // Now, check i'th row to remove to (i + 1)'th row to remove, until i is the
    // penultimate row.
    while (remove_ind < n_to_remove - 1)
    {
      size_t height = rows_to_remove[remove_ind + 1] -
          rows_to_remove[remove_ind] - 1;

      if (height > 0)
      {
        X_mod(span(cur_row, cur_row + height - 1), span::all) =
            X(span(rows_to_remove[remove_ind] + 1,
            rows_to_remove[remove_ind + 1] - 1), span::all);
        cur_row += height;
      }

      remove_ind++;
    }

    // Now that i is the last row to remove, check last row to remove to last
    // row.
    if (rows_to_remove[remove_ind] < n_rows - 1)
    {
      X_mod(span(cur_row, n_to_keep - 1), span::all) =
          X(span(rows_to_remove[remove_ind] + 1, n_rows - 1), span::all);
    }
  }
}
<commit_msg>In fact, remove the line entirely.<commit_after>/**
 * @file sparse_coding.cpp
 * @author Nishant Mehta
 *
 * Implementation of Sparse Coding with Dictionary Learning using l1 (LASSO) or
 * l1+l2 (Elastic Net) regularization.
 */
#include "sparse_coding.hpp"

using namespace std;
using namespace arma;
using namespace mlpack;
using namespace mlpack::regression;
using namespace mlpack::sparse_coding;

// TODO: parameterizable; options to methods?
#define OBJ_TOL 1e-2 // 1E-9
#define NEWTON_TOL 1e-6 // 1E-9

SparseCoding::SparseCoding(const mat& data,
                           const size_t atoms,
                           const double lambda1,
                           const double lambda2) :
    atoms(atoms),
    data(data),
    codes(mat(atoms, data.n_cols)),
    lambda1(lambda1),
    lambda2(lambda2)
{ /* Nothing left to do. */ }

// Always a not good decision!
void SparseCoding::RandomInitDictionary() {
  dictionary = randn(data.n_rows, atoms);

  for (size_t j = 0; j < atoms; ++j)
    dictionary.col(j) /= norm(dictionary.col(j), 2);
}

// The sensible heuristic.
void SparseCoding::DataDependentRandomInitDictionary()
{
  dictionary = mat(data.n_rows, atoms);
  for (size_t j = 0; j < atoms; ++j)
  {
    vec vecD_j = dictionary.unsafe_col(j);
    RandomAtom(vecD_j);
  }
}

void SparseCoding::RandomAtom(vec& atom)
{
  atom.zeros();
  const size_t nSeedAtoms = 3;
  for (size_t i = 0; i < nSeedAtoms; i++)
    atom += data.col(rand() % data.n_cols);

  atom /= norm(atom, 2);
}

void SparseCoding::DoSparseCoding(const size_t maxIterations)
{
  double lastObjVal = DBL_MAX;

  Log::Info << "Initial Coding Step." << endl;

  OptimizeCode();
  uvec adjacencies = find(codes);

  Log::Info << "  Sparsity level: "
      << 100.0 * ((double) (adjacencies.n_elem)) / ((double)
      (atoms * data.n_cols)) << "%" << endl;
  Log::Info << "  Objective value: " << Objective() << "." << endl;

  for (size_t t = 1; t != maxIterations; ++t)
  {
    Log::Info << "Iteration " << t << " of " << maxIterations << "." << endl;

    Log::Info << "Performing dictionary step... ";
    OptimizeDictionary(adjacencies);
    Log::Info << "objective value: " << Objective() << "." << endl;

    Log::Info << "Performing coding step..." << endl;
    OptimizeCode();
    adjacencies = find(codes);
    Log::Info << "  Sparsity level: "
        << 100.0 *
        ((double) (adjacencies.n_elem)) / ((double) (atoms * data.n_cols))
        << "%" << endl;

    double curObjVal = Objective();
    Log::Info << "  Objective value: " << curObjVal << "." << endl;

    double objValImprov = lastObjVal - curObjVal;
    Log::Info << "  Improvement: " << scientific << objValImprov << "." << endl;

    if (objValImprov < OBJ_TOL)
    {
      Log::Info << "Converged within tolerance " << OBJ_TOL << ".\n";
      break;
    }

    lastObjVal = curObjVal;
  }
}

void SparseCoding::OptimizeCode()
{
  // When using Cholesky version of LARS, this is correct even if lambda2 > 0.
  mat matGram = trans(dictionary) * dictionary;
  // mat matGram;
  // if(lambda2 > 0) {
  //   matGram = trans(dictionary) * dictionary + lambda2 * eye(atoms, atoms);
  // }
  // else {
  //   matGram = trans(dictionary) * dictionary;
  // }

  for (size_t i = 0; i < data.n_cols; ++i)
  {
    // Report progress.
    if ((i % 100) == 0)
      Log::Debug << "Optimization at point " << i << "." << endl;

    bool useCholesky = true;
    LARS lars(useCholesky, matGram, lambda1, lambda2);

    lars.DoLARS(dictionary, data.unsafe_col(i));

    vec beta;
    lars.Solution(beta);
    codes.col(i) = beta;
  }
}

void SparseCoding::OptimizeDictionary(const uvec& adjacencies)
{
  // Count the number of atomic neighbors for each point x^i.
  uvec neighborCounts = zeros<uvec>(data.n_cols, 1);

  if (adjacencies.n_elem > 0)
  {
    // This gets the column index.
    // TODO: is this integer division intentional?
    size_t curPointInd = (size_t) (adjacencies(0) / atoms);
    size_t curCount = 1;

    for (size_t l = 1; l < adjacencies.n_elem; ++l)
    {
      if ((size_t) (adjacencies(l) / atoms) == curPointInd)
      {
        ++curCount;
      }
      else
      {
        neighborCounts(curPointInd) = curCount;
        curPointInd = (size_t) (adjacencies(l) / atoms);
        curCount = 1;
      }
    }

    neighborCounts(curPointInd) = curCount;
  }

  // Handle the case of inactive atoms (atoms not used in the given coding).
  std::vector<size_t> inactiveAtoms;
  std::vector<size_t> activeAtoms;
  activeAtoms.reserve(atoms);

  for (size_t j = 0; j < atoms; ++j)
  {
    if (accu(codes.row(j) != 0) == 0)
      inactiveAtoms.push_back(j);
    else
      activeAtoms.push_back(j);
  }

  const size_t nActiveAtoms = activeAtoms.size();
  const size_t nInactiveAtoms = inactiveAtoms.size();

  // Efficient construction of Z restricted to active atoms.
  mat matActiveZ;
  if (inactiveAtoms.empty())
  {
    matActiveZ = codes;
  }
  else
  {
    uvec inactiveAtomsVec = conv_to<uvec>::from(inactiveAtoms);
    RemoveRows(codes, inactiveAtomsVec, matActiveZ);
  }

  uvec atomReverseLookup = uvec(atoms);
  for (size_t i = 0; i < nActiveAtoms; ++i)
    atomReverseLookup(activeAtoms[i]) = i;

  if (nInactiveAtoms > 0)
  {
    Log::Info << "There are " << nInactiveAtoms
        << " inactive atoms. They will be re-initialized randomly.\n";
  }

  Log::Debug << "Solving Dual via Newton's Method.\n";

  mat dictionaryEstimate;
  // Solve using Newton's method in the dual - note that the final dot
  // multiplication with inv(A) seems to be unavoidable. Although more
  // expensive, the code written this way (we use solve()) should be more
  // numerically stable than just using inv(A) for everything.
  vec dualVars = zeros<vec>(nActiveAtoms);

  //vec dualVars = 1e-14 * ones<vec>(nActiveAtoms);

  // Method used by feature sign code - fails miserably here.  Perhaps the
  // MATLAB optimizer fmincon does something clever?
  //vec dualVars = 10.0 * randu(nActiveAtoms, 1);

  //vec dualVars = diagvec(solve(dictionary, data * trans(codes))
  //    - codes * trans(codes));
  //for (size_t i = 0; i < dualVars.n_elem; i++)
  //  if (dualVars(i) < 0)
  //    dualVars(i) = 0;

  bool converged = false;
  mat codesXT = matActiveZ * trans(data);
  mat codesZT = matActiveZ * trans(matActiveZ);

  for (size_t t = 1; !converged; ++t)
  {
    mat A = codesZT + diagmat(dualVars);

    mat matAInvZXT = solve(A, codesXT);

    vec gradient = -(sum(square(matAInvZXT), 1) - ones<vec>(nActiveAtoms));

    mat hessian = -(-2 * (matAInvZXT * trans(matAInvZXT)) % inv(A));

    vec searchDirection = -solve(hessian, gradient);
    //vec searchDirection = -gradient;

    // Armijo line search.
    const double c = 1e-4;
    double alpha = 1.0;
    const double rho = 0.9;
    double sufficientDecrease = c * dot(gradient, searchDirection);

    /*
    {
      double sumDualVars = sum(dualVars);
      double fOld = -(-trace(trans(codesXT) * matAInvZXT) - sumDualVars);
      Log::Debug << "fOld = " << fOld << "." << endl;
      double fNew =
          -(-trace(trans(codesXT) * solve(codesZT +
          diagmat(dualVars + alpha * searchDirection), codesXT))
          - (sumDualVars + alpha * sum(searchDirection)) );
      Log::Debug << "fNew = " << fNew << "." << endl;
    }
    */

    double improvement;
    while (true)
    {
      // Calculate objective.
      double sumDualVars = sum(dualVars);
      double fOld = -(-trace(trans(codesXT) * matAInvZXT) - sumDualVars);
      double fNew = -(-trace(trans(codesXT) * solve(codesZT +
          diagmat(dualVars + alpha * searchDirection), codesXT)) -
          (sumDualVars + alpha * sum(searchDirection)));

      if (fNew <= fOld + alpha * sufficientDecrease)
      {
        searchDirection = alpha * searchDirection;
        improvement = fOld - fNew;
        break;
      }

      alpha *= rho;
    }

    // End of Armijo line search code.

    dualVars += searchDirection;
    double normGradient = norm(gradient, 2);
    Log::Debug << "Newton Method iteration " << t << ":" << endl;
    Log::Debug << "  Gradient norm: " << std::scientific << normGradient
        << "." << endl;
    Log::Debug << "  Improvement: " << std::scientific << improvement << ".\n";

    if (improvement < NEWTON_TOL)
      converged = true;
  }

  if (inactiveAtoms.empty())
  {
    dictionaryEstimate = trans(solve(codesZT + diagmat(dualVars), codesXT));
  }
  else
  {
    mat dictionaryActiveEstimate = trans(solve(codesZT + diagmat(dualVars),
        codesXT));
    dictionaryEstimate = zeros(data.n_rows, atoms);

    for (size_t i = 0; i < nActiveAtoms; ++i)
      dictionaryEstimate.col(activeAtoms[i]) = dictionaryActiveEstimate.col(i);

    for (size_t i = 0; i < nInactiveAtoms; ++i)
    {
      vec vecdictionaryi = dictionaryEstimate.unsafe_col(inactiveAtoms[i]);
      RandomAtom(vecdictionaryi);
    }
  }

  dictionary = dictionaryEstimate;
}

void SparseCoding::ProjectDictionary()
{
  for (size_t j = 0; j < atoms; j++)
  {
    double normD_j = norm(dictionary.col(j), 2);
    if ((normD_j > 1) && (normD_j - 1.0 > 1e-9))
    {
      Log::Warn << "Norm exceeded 1 by " << std::scientific << normD_j - 1.0
          << ".  Shrinking...\n";
      dictionary.col(j) /= normD_j;
    }
  }
}

double SparseCoding::Objective()
{
  double l11NormZ = sum(sum(abs(codes)));
  double froNormResidual = norm(data - dictionary * codes, "fro");

  if (lambda2 > 0)
  {
    double froNormZ = norm(codes, "fro");
    return 0.5 *
      (froNormResidual * froNormResidual + lambda2 * froNormZ * froNormZ) +
      lambda1 * l11NormZ;
  }
  else
  {
    return 0.5 * froNormResidual * froNormResidual + lambda1 * l11NormZ;
  }
}

void mlpack::sparse_coding::RemoveRows(const mat& X,
                                       uvec rows_to_remove,
                                       mat& X_mod)
{
  const size_t n_cols = X.n_cols;
  const size_t n_rows = X.n_rows;
  const size_t n_to_remove = rows_to_remove.n_elem;
  const size_t n_to_keep = n_rows - n_to_remove;

  if (n_to_remove == 0)
  {
    X_mod = X;
  }
  else
  {
    X_mod.set_size(n_to_keep, n_cols);

    size_t cur_row = 0;
    size_t remove_ind = 0;
    // First, check 0 to first row to remove.
    if (rows_to_remove(0) > 0)
    {
      // Note that this implies that n_rows > 1.
      size_t height = rows_to_remove(0);
      X_mod(span(cur_row, cur_row + height - 1), span::all) =
          X(span(0, rows_to_remove(0) - 1), span::all);
      cur_row += height;
    }
    // Now, check i'th row to remove to (i + 1)'th row to remove, until i is the
    // penultimate row.
    while (remove_ind < n_to_remove - 1)
    {
      size_t height = rows_to_remove[remove_ind + 1] -
          rows_to_remove[remove_ind] - 1;

      if (height > 0)
      {
        X_mod(span(cur_row, cur_row + height - 1), span::all) =
            X(span(rows_to_remove[remove_ind] + 1,
            rows_to_remove[remove_ind + 1] - 1), span::all);
        cur_row += height;
      }

      remove_ind++;
    }

    // Now that i is the last row to remove, check last row to remove to last
    // row.
    if (rows_to_remove[remove_ind] < n_rows - 1)
    {
      X_mod(span(cur_row, n_to_keep - 1), span::all) =
          X(span(rows_to_remove[remove_ind] + 1, n_rows - 1), span::all);
    }
  }
}
<|endoftext|>


<commit_before>#include "display.h"
#include "string.h"
#include "ports.h"
#include "assert.h"

namespace display {

Cell* videoram = reinterpret_cast<Cell*>(0xb8000);
int cursorX = 0;
int cursorY = 0;
u8 color;

u16 indexPort;
u16 dataPort;

void init() {
  // Find the base IO port for video from the BIOS Data Area. See
  // http://wiki.osdev.org/Memory_Map_%28x86%29#BIOS_Data_Area_.28BDA.29
  u16 baseIoPort = *reinterpret_cast<u16*>(0x0463);
  indexPort = baseIoPort;
  dataPort  = baseIoPort + 1;

  setColor(Color::LIGHT_GREY, Color::BLACK);
}

void setColor(Color fg, Color bg) {
  color = (u8(bg) << 4) | (u8(fg) & 0x0F);
}

void clearScreen() {
  for (int i = 0; i < consoleHeight * consoleWidth; i++)
    videoram[i] = Cell(color, ' ');

  cursorX = 0;
  cursorY = 0;
  updateCursor();
}

void scroll() {
  // Copy each line onto the one above it.
  for (int y = 0; y < consoleHeight - 1; y++)
    for (int x = 0; x < consoleWidth; x++)
      cellAt(x, y) = cellAt(x, y + 1);

  // Blank out the bottom line.
  for (int x = 0; x < consoleWidth; x++)
    cellAt(x, consoleHeight - 1) = Cell(color, ' ');
}

// Update the position of the blinking cursor on the screen.
void updateCursor() {
  const int position = cursorY * consoleWidth + cursorX;

  ports::outb(indexPort, cursorLowPort);
  ports::outb(dataPort, position >> 8);

  ports::outb(indexPort, cursorHighPort);
  ports::outb(dataPort, position);
}

Cell& cellAt(int x, int y) {
  return videoram[y * consoleWidth + x];
}

void printAt(char c, int x, int y) {
  cellAt(x, y) = Cell(color, c);
}

void print(char c) {
  switch(c) {
    case '\b':
      if (cursorX != 0)
        cursorX--;
      break;

    case '\t':
      // Align cursor_x to the next multiple of 8
      cursorX = cursorX - (cursorX % 8) + 8;
      break;

    case '\r':
      cursorX = 0;
      break;

    case '\n':
      cursorX = 0;
      cursorY++;
      break;

    default:
      printAt(c, cursorX, cursorY);
      cursorX++;
  }

  if (cursorX == consoleWidth) {
    cursorX = 0;
    cursorY++;
  }

  if (cursorY == consoleHeight) {
    cursorY--;
    scroll();
  }

  updateCursor();
}

void print(const char* str) {
  while (*str != '\0') {
    print(*str);
    str++;
  }
}

void printInt(u32 n, int radix) {
  assert(radix >= 2 && radix <= 36);

  // Support up to base 36.
  const char numerals[36] = {'0', '1', '2', '3', '4', '5', '6', '7',
    '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K',
    'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
    'Y', 'Z'};

  // 32 chars max in the smallest base (binary) for 32-bit integers.
  char buffer[32];

  if (n == 0) {
    print('0');
    return;
  }

  // Keep track of how long the number is.
  int i = 0;

  // Fill the buffer with digits from least significant to most
  // significant (reverse digit order).
  while (n != 0) {
    buffer[i] = numerals[n % radix];
    i++;
    n /= radix;
  }

  // Print the buffer in reverse order, since the digits are reversed.
  while (i != 0) {
    print(buffer[i - 1]);
    i--;
  }
}

void print(u32 n) {
  printInt(n, 10);
}

void print(i32 n) {
  if (n < 0) {
    print('-');
    // Use a cast instead of the expression -n, because -n can
    // overflow. E.g. if n == INT_MIN, -n returns a negative, because
    // -INT_MIN == INT_MIN.
    printInt(static_cast<u32>(n), 10);
  } else {
    printInt(n, 10);
  }
}

} // namespace display
<commit_msg>Minor style change.<commit_after>#include "display.h"
#include "string.h"
#include "ports.h"
#include "assert.h"

namespace display {

Cell* videoram = reinterpret_cast<Cell*>(0xb8000);
int cursorX = 0;
int cursorY = 0;
u8 color;

u16 indexPort;
u16 dataPort;

void init() {
  // Find the base IO port for video from the BIOS Data Area. See
  // http://wiki.osdev.org/Memory_Map_%28x86%29#BIOS_Data_Area_.28BDA.29
  u16 baseIoPort = *reinterpret_cast<u16*>(0x0463);
  indexPort = baseIoPort;
  dataPort  = baseIoPort + 1;

  setColor(Color::LIGHT_GREY, Color::BLACK);
}

void setColor(Color fg, Color bg) {
  color = (u8(bg) << 4) | (u8(fg) & 0x0F);
}

void clearScreen() {
  for (int i = 0; i < consoleHeight * consoleWidth; i++)
    videoram[i] = Cell(color, ' ');

  cursorX = 0;
  cursorY = 0;
  updateCursor();
}

void scroll() {
  // Copy each line onto the one above it.
  for (int y = 0; y < consoleHeight - 1; y++)
    for (int x = 0; x < consoleWidth; x++)
      cellAt(x, y) = cellAt(x, y + 1);

  // Blank out the bottom line.
  for (int x = 0; x < consoleWidth; x++)
    cellAt(x, consoleHeight - 1) = Cell(color, ' ');
}

// Update the position of the blinking cursor on the screen.
void updateCursor() {
  const int position = cursorY * consoleWidth + cursorX;

  ports::outb(indexPort, cursorLowPort);
  ports::outb(dataPort, position >> 8);

  ports::outb(indexPort, cursorHighPort);
  ports::outb(dataPort, position);
}

Cell& cellAt(int x, int y) {
  return videoram[y * consoleWidth + x];
}

void printAt(char c, int x, int y) {
  cellAt(x, y) = Cell(color, c);
}

void print(char c) {
  switch(c) {
    case '\b':
      if (cursorX != 0) {
        cursorX--;
      }
      break;

    case '\t':
      // Align cursor_x to the next multiple of 8
      cursorX = cursorX - (cursorX % 8) + 8;
      break;

    case '\r':
      cursorX = 0;
      break;

    case '\n':
      cursorX = 0;
      cursorY++;
      break;

    default:
      printAt(c, cursorX, cursorY);
      cursorX++;
  }

  if (cursorX == consoleWidth) {
    cursorX = 0;
    cursorY++;
  }

  if (cursorY == consoleHeight) {
    cursorY--;
    scroll();
  }

  updateCursor();
}

void print(const char* str) {
  while (*str != '\0') {
    print(*str);
    str++;
  }
}

void printInt(u32 n, int radix) {
  assert(radix >= 2 && radix <= 36);

  // Support up to base 36.
  const char numerals[36] = {'0', '1', '2', '3', '4', '5', '6', '7',
    '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K',
    'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
    'Y', 'Z'};

  // 32 chars max in the smallest base (binary) for 32-bit integers.
  char buffer[32];

  if (n == 0) {
    print('0');
    return;
  }

  // Keep track of how long the number is.
  int i = 0;

  // Fill the buffer with digits from least significant to most
  // significant (reverse digit order).
  while (n != 0) {
    buffer[i] = numerals[n % radix];
    i++;
    n /= radix;
  }

  // Print the buffer in reverse order, since the digits are reversed.
  while (i != 0) {
    print(buffer[i - 1]);
    i--;
  }
}

void print(u32 n) {
  printInt(n, 10);
}

void print(i32 n) {
  if (n < 0) {
    print('-');
    // Use a cast instead of the expression -n, because -n can
    // overflow. E.g. if n == INT_MIN, -n returns a negative, because
    // -INT_MIN == INT_MIN.
    printInt(static_cast<u32>(n), 10);
  } else {
    printInt(n, 10);
  }
}

} // namespace display
<|endoftext|>


<commit_before>#pragma once

#include <iostream>
#include <memory>
#include <sstream>
#include <mutex>

#include "threading/threading.hpp"

namespace nest {
namespace mc {
namespace util {

bool failed_assertion(const char* assertion, const char* file, int line, const char* func);
std::ostream &debug_emit_trace_leader(std::ostream& out, const char* file, int line, const char* varlist);

inline void debug_emit(std::ostream& out) {
    out << "\n";
}

template <typename Head, typename... Tail>
void debug_emit(std::ostream& out, const Head& head, const Tail&... tail) {
    out << head;
    if (sizeof...(tail)) {
        out << ", ";
    }
    debug_emit(out, tail...);
}

extern std::mutex global_debug_cerr_mutex;

template <typename... Args>
void debug_emit_trace(const char* file, int line, const char* varlist, const Args&... args) {
    constexpr bool multithreaded = nest::mc::threading::multithreaded();

    std::unique_ptr<std::ostream> buffer;
    std::ostream* out = &std::cerr;

    if (multithreaded) {
        buffer.reset(new std::stringstream());
        out = buffer.get();
    }

    debug_emit_trace_leader(*out, file, line, varlist);
    debug_emit(*out, args...);

    if (multithreaded) {
        std::lock_guard<std::mutex> guard(global_debug_cerr_mutex);
        std::cerr << out->rdbuf();
        std::cerr.flush();
    }
    else {
        out->flush();
    }
}

} // namespace util
} // namespace mc
} // namespace nest

#ifdef WITH_TRACE
    #define TRACE(vars...) nest::mc::util::debug_emit_trace(__FILE__, __LINE__, #vars, ##vars)
#else
    #define TRACE(...)
#endif

#ifdef WITH_ASSERTIONS
    #ifdef __GNUC__
        #define DEBUG_FUNCTION_NAME __PRETTY_FUNCTION__
    #else
        #define DEBUG_FUNCTION_NAME __func__
    #endif

    #define EXPECTS(condition) \
       (void)((condition) || \
       nest::mc::util::failed_assertion(#condition, __FILE__, __LINE__, DEBUG_FUNCTION_NAME))
#else
    #define EXPECTS(condition)
#endif // def WITH_ASSERTIONS
<commit_msg>Further simplification of debug_emit_trace logic.<commit_after>#pragma once

#include <iostream>
#include <sstream>
#include <mutex>

#include "threading/threading.hpp"

namespace nest {
namespace mc {
namespace util {

bool failed_assertion(const char* assertion, const char* file, int line, const char* func);
std::ostream &debug_emit_trace_leader(std::ostream& out, const char* file, int line, const char* varlist);

inline void debug_emit(std::ostream& out) {
    out << "\n";
}

template <typename Head, typename... Tail>
void debug_emit(std::ostream& out, const Head& head, const Tail&... tail) {
    out << head;
    if (sizeof...(tail)) {
        out << ", ";
    }
    debug_emit(out, tail...);
}

extern std::mutex global_debug_cerr_mutex;

template <typename... Args>
void debug_emit_trace(const char* file, int line, const char* varlist, const Args&... args) {
    if (nest::mc::threading::multithreaded()) {
        std::stringstream buffer;

        debug_emit_trace_leader(buffer, file, line, varlist);
        debug_emit(buffer, args...);

        std::lock_guard<std::mutex> guard(global_debug_cerr_mutex);
        std::cerr << buffer.rdbuf();
        std::cerr.flush();
    }
    else {
        debug_emit_trace_leader(std::cerr, file, line, varlist);
        debug_emit(std::cerr, args...);
        std::cerr.flush();
    }
}

} // namespace util
} // namespace mc
} // namespace nest

#ifdef WITH_TRACE
    #define TRACE(vars...) nest::mc::util::debug_emit_trace(__FILE__, __LINE__, #vars, ##vars)
#else
    #define TRACE(...)
#endif

#ifdef WITH_ASSERTIONS
    #ifdef __GNUC__
        #define DEBUG_FUNCTION_NAME __PRETTY_FUNCTION__
    #else
        #define DEBUG_FUNCTION_NAME __func__
    #endif

    #define EXPECTS(condition) \
       (void)((condition) || \
       nest::mc::util::failed_assertion(#condition, __FILE__, __LINE__, DEBUG_FUNCTION_NAME))
#else
    #define EXPECTS(condition)
#endif // def WITH_ASSERTIONS
<|endoftext|>


<commit_before>/*!
 * \file    SimulationNBodyMPI.hxx
 * \brief   Abstract n-body collision less simulation class for MPI computations (between many nodes).
 * \author  A. Cassagne
 * \date    2014
 *
 * \section LICENSE
 * This file is under MIT license (https://opensource.org/licenses/MIT).
 */
#ifdef USE_MPI
#include <cmath>
#include <limits>
#include <string>
#include <cassert>
#include <fstream>
#include <iostream>

#ifdef _OPENMP
#include <omp.h>
#else
#ifndef NO_OMP
#define NO_OMP
inline void omp_set_num_threads(int) {           }
inline int  omp_get_num_threads(   ) { return 1; }
inline int  omp_get_max_threads(   ) { return 1; }
inline int  omp_get_thread_num (   ) { return 0; }
#endif
#endif

#include "../../../common/utils/ToMPIDatatype.h"

#include "SimulationNBodyMPI.h"

MPIRank get_rank()
{
	MPIRank rank;
	MPI_Comm_rank(MPI_COMM_WORLD, &rank)
	return rank;
}
MPISize get_size()
{
	MPISize size;
	MPI_Comm_size(MPI_COMM_WORLD, &size)
	return size;
}

template <typename T>
SimulationNBodyMPI<T>::SimulationNBodyMPI(const unsigned long nBodies)
	: SimulationNBody<T>(new Bodies<T>(nBodies, get_rank() * nBodies +1)),
	  neighborBodies    (NULL),
	  MPISize           (get_size()),
	  MPIRank           (get_rank()),
	  MPIBodiesBuffers  {this->bodies->getN(), this->bodies->getN()}
{
	this->init();
}

template <typename T>
SimulationNBodyMPI<T>::SimulationNBodyMPI(const std::string inputFileName)
	: SimulationNBody<T>(new Bodies<T>(inputFileName)),
	  neighborBodies    (NULL),
	  MPISize           (get_size()),
	  MPIRank           (get_rank()),
	  MPIBodiesBuffers  {this->bodies->getN(), this->bodies->getN()}
{
	this->init();
}

template <typename T>
void SimulationNBodyMPI<T>::init()
{
	const unsigned long n = this->bodies->getN() + this->bodies->getPadding();
	MPI_Datatype MPIType = ToMPIDatatype<T>::value();

	// who is next or previous MPI process ?
	int MPIPrevRank = (this->MPIRank == 0) ? this->MPISize -1 : this->MPIRank -1;
	int MPINextRank = (this->MPIRank +1) % this->MPISize;

	Bodies<T> *bBuffs[] = {&this->MPIBodiesBuffers[0], &this->MPIBodiesBuffers[1]};

	// send number of bodies
	MPI_Send_init(&bBuffs[0]->n,            1, MPI_LONG, MPINextRank, 0, MPI_COMM_WORLD, &this->MPIRequests[0][0]);
	MPI_Send_init(&bBuffs[1]->n,            1, MPI_LONG, MPINextRank, 1, MPI_COMM_WORLD, &this->MPIRequests[1][0]);
	// receive number of bodies
	MPI_Recv_init(&bBuffs[1]->n,            1, MPI_LONG, MPIPrevRank, 0, MPI_COMM_WORLD, &this->MPIRequests[0][1]);
	MPI_Recv_init(&bBuffs[0]->n,            1, MPI_LONG, MPIPrevRank, 1, MPI_COMM_WORLD, &this->MPIRequests[1][1]);

	// send masses
	MPI_Send_init(bBuffs[0]->masses,        n, MPIType,  MPINextRank, 2, MPI_COMM_WORLD, &this->MPIRequests[0][2]);
	MPI_Send_init(bBuffs[1]->masses,        n, MPIType,  MPINextRank, 3, MPI_COMM_WORLD, &this->MPIRequests[1][2]);
	// receive masses
	MPI_Recv_init(bBuffs[1]->masses,        n, MPIType,  MPIPrevRank, 2, MPI_COMM_WORLD, &this->MPIRequests[0][3]);
	MPI_Recv_init(bBuffs[0]->masses,        n, MPIType,  MPIPrevRank, 3, MPI_COMM_WORLD, &this->MPIRequests[1][3]);

	// send radiuses
	MPI_Send_init(bBuffs[0]->radiuses,      n, MPIType,  MPINextRank, 4, MPI_COMM_WORLD, &this->MPIRequests[0][4]);
	MPI_Send_init(bBuffs[1]->radiuses,      n, MPIType,  MPINextRank, 5, MPI_COMM_WORLD, &this->MPIRequests[1][4]);
	// receive radiuses
	MPI_Recv_init(bBuffs[1]->radiuses,      n, MPIType,  MPIPrevRank, 4, MPI_COMM_WORLD, &this->MPIRequests[0][5]);
	MPI_Recv_init(bBuffs[0]->radiuses,      n, MPIType,  MPIPrevRank, 5, MPI_COMM_WORLD, &this->MPIRequests[1][5]);

	// send positions.x
	MPI_Send_init(bBuffs[0]->positions.x,   n, MPIType,  MPINextRank, 6, MPI_COMM_WORLD, &this->MPIRequests[0][6]);
	MPI_Send_init(bBuffs[1]->positions.x,   n, MPIType,  MPINextRank, 7, MPI_COMM_WORLD, &this->MPIRequests[1][6]);
	// receive positions.x
	MPI_Recv_init(bBuffs[1]->positions.x,   n, MPIType,  MPIPrevRank, 6, MPI_COMM_WORLD, &this->MPIRequests[0][7]);
	MPI_Recv_init(bBuffs[0]->positions.x,   n, MPIType,  MPIPrevRank, 7, MPI_COMM_WORLD, &this->MPIRequests[1][7]);

	// send positions.y
	MPI_Send_init(bBuffs[0]->positions.y,   n, MPIType,  MPINextRank, 8, MPI_COMM_WORLD, &this->MPIRequests[0][8]);
	MPI_Send_init(bBuffs[1]->positions.y,   n, MPIType,  MPINextRank, 9, MPI_COMM_WORLD, &this->MPIRequests[1][8]);
	// receive positions.y
	MPI_Recv_init(bBuffs[1]->positions.y,   n, MPIType,  MPIPrevRank, 8, MPI_COMM_WORLD, &this->MPIRequests[0][9]);
	MPI_Recv_init(bBuffs[0]->positions.y,   n, MPIType,  MPIPrevRank, 9, MPI_COMM_WORLD, &this->MPIRequests[1][9]);

	// send positions.z
	MPI_Send_init(bBuffs[0]->positions.z,  n, MPIType, MPINextRank, 10, MPI_COMM_WORLD, &this->MPIRequests[0][10]);
	MPI_Send_init(bBuffs[1]->positions.z,  n, MPIType, MPINextRank, 11, MPI_COMM_WORLD, &this->MPIRequests[1][10]);
	// receive positions.z
	MPI_Recv_init(bBuffs[1]->positions.z,  n, MPIType, MPIPrevRank, 10, MPI_COMM_WORLD, &this->MPIRequests[0][11]);
	MPI_Recv_init(bBuffs[0]->positions.z,  n, MPIType, MPIPrevRank, 11, MPI_COMM_WORLD, &this->MPIRequests[1][11]);

	// send velocities.x
	MPI_Send_init(bBuffs[0]->velocities.x, n, MPIType, MPINextRank, 12, MPI_COMM_WORLD, &this->MPIRequests[0][12]);
	MPI_Send_init(bBuffs[1]->velocities.x, n, MPIType, MPINextRank, 13, MPI_COMM_WORLD, &this->MPIRequests[1][12]);
	// receive velocities.x
	MPI_Recv_init(bBuffs[1]->velocities.x, n, MPIType, MPIPrevRank, 12, MPI_COMM_WORLD, &this->MPIRequests[0][13]);
	MPI_Recv_init(bBuffs[0]->velocities.x, n, MPIType, MPIPrevRank, 13, MPI_COMM_WORLD, &this->MPIRequests[1][13]);

	// send velocities.y
	MPI_Send_init(bBuffs[0]->velocities.y, n, MPIType, MPINextRank, 14, MPI_COMM_WORLD, &this->MPIRequests[0][14]);
	MPI_Send_init(bBuffs[1]->velocities.y, n, MPIType, MPINextRank, 15, MPI_COMM_WORLD, &this->MPIRequests[1][14]);
	// receive velocities.y
	MPI_Recv_init(bBuffs[1]->velocities.y, n, MPIType, MPIPrevRank, 14, MPI_COMM_WORLD, &this->MPIRequests[0][15]);
	MPI_Recv_init(bBuffs[0]->velocities.y, n, MPIType, MPIPrevRank, 15, MPI_COMM_WORLD, &this->MPIRequests[1][15]);

	// send velocities.z
	MPI_Send_init(bBuffs[0]->velocities.z, n, MPIType, MPINextRank, 16, MPI_COMM_WORLD, &this->MPIRequests[0][16]);
	MPI_Send_init(bBuffs[1]->velocities.z, n, MPIType, MPINextRank, 17, MPI_COMM_WORLD, &this->MPIRequests[1][16]);
	// receive velocities.z
	MPI_Recv_init(bBuffs[1]->velocities.z, n, MPIType, MPIPrevRank, 16, MPI_COMM_WORLD, &this->MPIRequests[0][17]);
	MPI_Recv_init(bBuffs[0]->velocities.z, n, MPIType, MPIPrevRank, 17, MPI_COMM_WORLD, &this->MPIRequests[1][17]);
}

template <typename T>
SimulationNBodyMPI<T>::~SimulationNBodyMPI()
{
	delete this->bodies;
}

template <typename T>
void SimulationNBodyMPI<T>::computeOneIteration()
{
	this->MPIBodiesBuffers[0].hardCopy(*this->bodies);

	this->initIteration();

	// compute bodies acceleration ------------------------------------------------------------------------------------
	MPI_Prequest::Startall(2*9, this->MPIRequests[0]);

	this->computeLocalBodiesAcceleration();

	for(int iStep = 1; iStep < this->MPISize; ++iStep)
	{
		MPI_Request::Waitall(2*9, this->MPIRequests[(iStep -1) % 2]);

		this->neighborBodies = &this->MPIBodiesBuffers[iStep % 2];

		this->computeNeighborBodiesAcceleration();

		if(iStep < this->MPISize -1)
			MPI_Prequest::Startall(2*9, this->MPIRequests[iStep % 2]);
	}
	// ----------------------------------------------------------------------------------------------------------------

	// find time step -------------------------------------------------------------------------------------------------
	if(!this->dtConstant)
		this->findTimeStep();
	// ----------------------------------------------------------------------------------------------------------------

	// update positions and velocities --------------------------------------------------------------------------------
	this->bodies->updatePositionsAndVelocities(this->accelerations, this->dt);
	// ----------------------------------------------------------------------------------------------------------------
}

template <typename T>
void SimulationNBodyMPI<T>::findTimeStep()
{
	// TODO: be careful with the V1Intrinsics version: with fake bodies added at the end of the last vector, the
	//       dynamic time step is broken.
	//       It is necessary to launch the simulation with a number of bodies multiple of mipp::vectorSize<T>()!
	if(!this->dtConstant)
	{
		T localDt = std::numeric_limits<T>::infinity();
		for(unsigned long iBody = 0; iBody < this->bodies->getN(); iBody++)
			localDt = std::min(localDt, this->computeTimeStep(iBody));

		MPI_Allreduce(&localDt, &this->dt, 1, ToMPIDatatype<T>::value(), MPI_MIN, MPI_COMM_WORLD);

		if(this->dt < this->minDt)
			this->dt = this->minDt;
	}
}

// ==================================================================================== explicit template instantiation
#ifdef NBODY_DOUBLE
template class SimulationNBodyMPI<double>;
#else
template class SimulationNBodyMPI<float>;
#endif
// ==================================================================================== explicit template instantiation
#endif
<commit_msg>WIP C bindings<commit_after>/*!
 * \file    SimulationNBodyMPI.hxx
 * \brief   Abstract n-body collision less simulation class for MPI computations (between many nodes).
 * \author  A. Cassagne
 * \date    2014
 *
 * \section LICENSE
 * This file is under MIT license (https://opensource.org/licenses/MIT).
 */
#ifdef USE_MPI
#include <cmath>
#include <limits>
#include <string>
#include <cassert>
#include <fstream>
#include <iostream>

#ifdef _OPENMP
#include <omp.h>
#else
#ifndef NO_OMP
#define NO_OMP
inline void omp_set_num_threads(int) {           }
inline int  omp_get_num_threads(   ) { return 1; }
inline int  omp_get_max_threads(   ) { return 1; }
inline int  omp_get_thread_num (   ) { return 0; }
#endif
#endif

#include "../../../common/utils/ToMPIDatatype.h"

#include "SimulationNBodyMPI.h"

int get_rank()
{
	int rank;
	MPI_Comm_rank(MPI_COMM_WORLD, &rank);
	return rank;
}
int get_size()
{
	int size;
	MPI_Comm_size(MPI_COMM_WORLD, &size);
	return size;
}

template <typename T>
SimulationNBodyMPI<T>::SimulationNBodyMPI(const unsigned long nBodies)
	: SimulationNBody<T>(new Bodies<T>(nBodies, get_rank() * nBodies +1)),
	  neighborBodies    (NULL),
	  MPISize           (get_size()),
	  MPIRank           (get_rank()),
	  MPIBodiesBuffers  {this->bodies->getN(), this->bodies->getN()}
{
	this->init();
}

template <typename T>
SimulationNBodyMPI<T>::SimulationNBodyMPI(const std::string inputFileName)
	: SimulationNBody<T>(new Bodies<T>(inputFileName)),
	  neighborBodies    (NULL),
	  MPISize           (get_size()),
	  MPIRank           (get_rank()),
	  MPIBodiesBuffers  {this->bodies->getN(), this->bodies->getN()}
{
	this->init();
}

template <typename T>
void SimulationNBodyMPI<T>::init()
{
	const unsigned long n = this->bodies->getN() + this->bodies->getPadding();
	MPI_Datatype MPIType = ToMPIDatatype<T>::value();

	// who is next or previous MPI process ?
	int MPIPrevRank = (this->MPIRank == 0) ? this->MPISize -1 : this->MPIRank -1;
	int MPINextRank = (this->MPIRank +1) % this->MPISize;

	Bodies<T> *bBuffs[] = {&this->MPIBodiesBuffers[0], &this->MPIBodiesBuffers[1]};

	// send number of bodies
	MPI_Send_init(&bBuffs[0]->n,            1, MPI_LONG, MPINextRank, 0, MPI_COMM_WORLD, &this->MPIRequests[0][0]);
	MPI_Send_init(&bBuffs[1]->n,            1, MPI_LONG, MPINextRank, 1, MPI_COMM_WORLD, &this->MPIRequests[1][0]);
	// receive number of bodies
	MPI_Recv_init(&bBuffs[1]->n,            1, MPI_LONG, MPIPrevRank, 0, MPI_COMM_WORLD, &this->MPIRequests[0][1]);
	MPI_Recv_init(&bBuffs[0]->n,            1, MPI_LONG, MPIPrevRank, 1, MPI_COMM_WORLD, &this->MPIRequests[1][1]);

	// send masses
	MPI_Send_init(bBuffs[0]->masses,        n, MPIType,  MPINextRank, 2, MPI_COMM_WORLD, &this->MPIRequests[0][2]);
	MPI_Send_init(bBuffs[1]->masses,        n, MPIType,  MPINextRank, 3, MPI_COMM_WORLD, &this->MPIRequests[1][2]);
	// receive masses
	MPI_Recv_init(bBuffs[1]->masses,        n, MPIType,  MPIPrevRank, 2, MPI_COMM_WORLD, &this->MPIRequests[0][3]);
	MPI_Recv_init(bBuffs[0]->masses,        n, MPIType,  MPIPrevRank, 3, MPI_COMM_WORLD, &this->MPIRequests[1][3]);

	// send radiuses
	MPI_Send_init(bBuffs[0]->radiuses,      n, MPIType,  MPINextRank, 4, MPI_COMM_WORLD, &this->MPIRequests[0][4]);
	MPI_Send_init(bBuffs[1]->radiuses,      n, MPIType,  MPINextRank, 5, MPI_COMM_WORLD, &this->MPIRequests[1][4]);
	// receive radiuses
	MPI_Recv_init(bBuffs[1]->radiuses,      n, MPIType,  MPIPrevRank, 4, MPI_COMM_WORLD, &this->MPIRequests[0][5]);
	MPI_Recv_init(bBuffs[0]->radiuses,      n, MPIType,  MPIPrevRank, 5, MPI_COMM_WORLD, &this->MPIRequests[1][5]);

	// send positions.x
	MPI_Send_init(bBuffs[0]->positions.x,   n, MPIType,  MPINextRank, 6, MPI_COMM_WORLD, &this->MPIRequests[0][6]);
	MPI_Send_init(bBuffs[1]->positions.x,   n, MPIType,  MPINextRank, 7, MPI_COMM_WORLD, &this->MPIRequests[1][6]);
	// receive positions.x
	MPI_Recv_init(bBuffs[1]->positions.x,   n, MPIType,  MPIPrevRank, 6, MPI_COMM_WORLD, &this->MPIRequests[0][7]);
	MPI_Recv_init(bBuffs[0]->positions.x,   n, MPIType,  MPIPrevRank, 7, MPI_COMM_WORLD, &this->MPIRequests[1][7]);

	// send positions.y
	MPI_Send_init(bBuffs[0]->positions.y,   n, MPIType,  MPINextRank, 8, MPI_COMM_WORLD, &this->MPIRequests[0][8]);
	MPI_Send_init(bBuffs[1]->positions.y,   n, MPIType,  MPINextRank, 9, MPI_COMM_WORLD, &this->MPIRequests[1][8]);
	// receive positions.y
	MPI_Recv_init(bBuffs[1]->positions.y,   n, MPIType,  MPIPrevRank, 8, MPI_COMM_WORLD, &this->MPIRequests[0][9]);
	MPI_Recv_init(bBuffs[0]->positions.y,   n, MPIType,  MPIPrevRank, 9, MPI_COMM_WORLD, &this->MPIRequests[1][9]);

	// send positions.z
	MPI_Send_init(bBuffs[0]->positions.z,  n, MPIType, MPINextRank, 10, MPI_COMM_WORLD, &this->MPIRequests[0][10]);
	MPI_Send_init(bBuffs[1]->positions.z,  n, MPIType, MPINextRank, 11, MPI_COMM_WORLD, &this->MPIRequests[1][10]);
	// receive positions.z
	MPI_Recv_init(bBuffs[1]->positions.z,  n, MPIType, MPIPrevRank, 10, MPI_COMM_WORLD, &this->MPIRequests[0][11]);
	MPI_Recv_init(bBuffs[0]->positions.z,  n, MPIType, MPIPrevRank, 11, MPI_COMM_WORLD, &this->MPIRequests[1][11]);

	// send velocities.x
	MPI_Send_init(bBuffs[0]->velocities.x, n, MPIType, MPINextRank, 12, MPI_COMM_WORLD, &this->MPIRequests[0][12]);
	MPI_Send_init(bBuffs[1]->velocities.x, n, MPIType, MPINextRank, 13, MPI_COMM_WORLD, &this->MPIRequests[1][12]);
	// receive velocities.x
	MPI_Recv_init(bBuffs[1]->velocities.x, n, MPIType, MPIPrevRank, 12, MPI_COMM_WORLD, &this->MPIRequests[0][13]);
	MPI_Recv_init(bBuffs[0]->velocities.x, n, MPIType, MPIPrevRank, 13, MPI_COMM_WORLD, &this->MPIRequests[1][13]);

	// send velocities.y
	MPI_Send_init(bBuffs[0]->velocities.y, n, MPIType, MPINextRank, 14, MPI_COMM_WORLD, &this->MPIRequests[0][14]);
	MPI_Send_init(bBuffs[1]->velocities.y, n, MPIType, MPINextRank, 15, MPI_COMM_WORLD, &this->MPIRequests[1][14]);
	// receive velocities.y
	MPI_Recv_init(bBuffs[1]->velocities.y, n, MPIType, MPIPrevRank, 14, MPI_COMM_WORLD, &this->MPIRequests[0][15]);
	MPI_Recv_init(bBuffs[0]->velocities.y, n, MPIType, MPIPrevRank, 15, MPI_COMM_WORLD, &this->MPIRequests[1][15]);

	// send velocities.z
	MPI_Send_init(bBuffs[0]->velocities.z, n, MPIType, MPINextRank, 16, MPI_COMM_WORLD, &this->MPIRequests[0][16]);
	MPI_Send_init(bBuffs[1]->velocities.z, n, MPIType, MPINextRank, 17, MPI_COMM_WORLD, &this->MPIRequests[1][16]);
	// receive velocities.z
	MPI_Recv_init(bBuffs[1]->velocities.z, n, MPIType, MPIPrevRank, 16, MPI_COMM_WORLD, &this->MPIRequests[0][17]);
	MPI_Recv_init(bBuffs[0]->velocities.z, n, MPIType, MPIPrevRank, 17, MPI_COMM_WORLD, &this->MPIRequests[1][17]);
}

template <typename T>
SimulationNBodyMPI<T>::~SimulationNBodyMPI()
{
	delete this->bodies;
}

template <typename T>
void SimulationNBodyMPI<T>::computeOneIteration()
{
	this->MPIBodiesBuffers[0].hardCopy(*this->bodies);

	this->initIteration();

	// compute bodies acceleration ------------------------------------------------------------------------------------
	MPI_Startall(2*9, this->MPIRequests[0]);

	this->computeLocalBodiesAcceleration();

	for(int iStep = 1; iStep < this->MPISize; ++iStep)
	{
		MPI_Waitall(2*9, this->MPIRequests[(iStep -1) % 2], MPI_STATUSES_IGNORE);

		this->neighborBodies = &this->MPIBodiesBuffers[iStep % 2];

		this->computeNeighborBodiesAcceleration();

		if(iStep < this->MPISize -1)
			MPI_Startall(2*9, this->MPIRequests[iStep % 2]);
	}
	// ----------------------------------------------------------------------------------------------------------------

	// find time step -------------------------------------------------------------------------------------------------
	if(!this->dtConstant)
		this->findTimeStep();
	// ----------------------------------------------------------------------------------------------------------------

	// update positions and velocities --------------------------------------------------------------------------------
	this->bodies->updatePositionsAndVelocities(this->accelerations, this->dt);
	// ----------------------------------------------------------------------------------------------------------------
}

template <typename T>
void SimulationNBodyMPI<T>::findTimeStep()
{
	// TODO: be careful with the V1Intrinsics version: with fake bodies added at the end of the last vector, the
	//       dynamic time step is broken.
	//       It is necessary to launch the simulation with a number of bodies multiple of mipp::vectorSize<T>()!
	if(!this->dtConstant)
	{
		T localDt = std::numeric_limits<T>::infinity();
		for(unsigned long iBody = 0; iBody < this->bodies->getN(); iBody++)
			localDt = std::min(localDt, this->computeTimeStep(iBody));

		MPI_Allreduce(&localDt, &this->dt, 1, ToMPIDatatype<T>::value(), MPI_MIN, MPI_COMM_WORLD);

		if(this->dt < this->minDt)
			this->dt = this->minDt;
	}
}

// ==================================================================================== explicit template instantiation
#ifdef NBODY_DOUBLE
template class SimulationNBodyMPI<double>;
#else
template class SimulationNBodyMPI<float>;
#endif
// ==================================================================================== explicit template instantiation
#endif
<|endoftext|>


<commit_before>// Adobe Leaked Email Chcker
// Date: December 2013
// Author: Jervis Muindi

// Standard Lib Header
#include <iostream>

// Custom Common Code
//#include "common/base/flags.h"
//#include "common/base/init.h"
//#include "common/log/log.h"
#include "common/test/test.h"
#include "common/strings/strutil.h"

#include "alec.h"

using namespace alec;

TEST(Alec, ParseLine) {
  string line = "000000006-|--|-person6@yahoo.com-|-DGM2c/HbXTIkDDM5y6e6/lQ==-|-same|--";
  Credential expected_cred;
  Credential actual_cred;

  expected_cred.rec_id = "000000006";
  expected_cred.username = "";
  expected_cred.email = "person6@yahoo.com";
  expected_cred.hash = "DGM2c/HbXTIkDDM5y6e6/lQ==";
  expected_cred.hint = "same";

  EXPECT_TRUE(CredentialReader::ParseLine(line, &actual_cred))
    << "Failed to Parse Line: " << line;
  EXPECT_TRUE ( expected_cred == actual_cred )
    << "Expected Cred:\n" << expected_cred
    << "**********\nBut Got:\n" << actual_cred;
}

int main(int argc, char **argv) {
  testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}
<commit_msg>add test for reading from file<commit_after>// Adobe Leaked Email Chcker
// Date: December 2013
// Author: Jervis Muindi

// Standard Lib Header
#include <iostream>

// Custom Common Code
//#include "common/base/flags.h"
//#include "common/base/init.h"
//#include "common/log/log.h"
#include "common/test/test.h"
#include "common/strings/strutil.h"

#include "alec.h"

using namespace alec;

TEST(Alec, ReadCredentialFile) {
  CredentialReader reader("sample_cred.txt");
  EXPECT_FALSE(reader.Done());

  Credential expected_cred;
  expected_cred.rec_id = "000000006";
  expected_cred.username = "";
  expected_cred.email = "person6@yahoo.com";
  expected_cred.hash = "DGM2c/HbXTIkDDM5y6e6/lQ==";
  expected_cred.hint = "same";

  Credential cred;
  while (!reader.Done()) {
    EXPECT_TRUE(reader.NextCredential(&cred)) << "Failed to get next credential";
    EXPECT_TRUE ( expected_cred == cred )
      << "Expected Cred:\n" << expected_cred
      << "**********\nBut Got:\n" << cred;
  }
}

TEST(Alec, ParseLine) {
  string line = "000000006-|--|-person6@yahoo.com-|-DGM2c/HbXTIkDDM5y6e6/lQ==-|-same|--";
  Credential expected_cred;
  Credential actual_cred;

  expected_cred.rec_id = "000000006";
  expected_cred.username = "";
  expected_cred.email = "person6@yahoo.com";
  expected_cred.hash = "DGM2c/HbXTIkDDM5y6e6/lQ==";
  expected_cred.hint = "same";

  EXPECT_TRUE(CredentialReader::ParseLine(line, &actual_cred))
    << "Failed to Parse Line: " << line;
  EXPECT_TRUE ( expected_cred == actual_cred )
    << "Expected Cred:\n" << expected_cred
    << "**********\nBut Got:\n" << actual_cred;
}

int main(int argc, char **argv) {
  testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}
<|endoftext|>