<commit_before>/////////////////////////////////////////////////////////////////////
// = NMatrix
//
// A linear algebra library for scientific computation in Ruby.
// NMatrix is part of SciRuby.
//
// NMatrix was originally inspired by and derived from NArray, by
// Masahiro Tanaka: http://narray.rubyforge.org
//
// == Copyright Information
//
// SciRuby is Copyright (c) 2010 - 2012, Ruby Science Foundation
// NMatrix is Copyright (c) 2012, Ruby Science Foundation
//
// Please see LICENSE.txt for additional copyright notices.
//
// == Contributing
//
// By contributing source code to SciRuby, you agree to be bound by
// our Contributor Agreement:
//
// * https://github.com/SciRuby/sciruby/wiki/Contributor-Agreement
//
// == storage.c
//
// Code that is used by or involves more then one storage type.

/*
 * Standard Includes
 */

/*
 * Project Includes
 */

#include "data/data.h"

#include "storage.h"

/*
 * Macros
 */

/*
 * Global Variables
 */

const char* const STYPE_NAMES[NUM_STYPES] {
	"dense",
	"list",
	"yale"
}

/*
 * Forward Declarations
 */

/*
 * Functions
 */

<commit_msg>Moved code from storage_templates.cpp into this file now that everything is being compiled by g++.<commit_after>/////////////////////////////////////////////////////////////////////
// = NMatrix
//
// A linear algebra library for scientific computation in Ruby.
// NMatrix is part of SciRuby.
//
// NMatrix was originally inspired by and derived from NArray, by
// Masahiro Tanaka: http://narray.rubyforge.org
//
// == Copyright Information
//
// SciRuby is Copyright (c) 2010 - 2012, Ruby Science Foundation
// NMatrix is Copyright (c) 2012, Ruby Science Foundation
//
// Please see LICENSE.txt for additional copyright notices.
//
// == Contributing
//
// By contributing source code to SciRuby, you agree to be bound by
// our Contributor Agreement:
//
// * https://github.com/SciRuby/sciruby/wiki/Contributor-Agreement
//
// == storage.cpp
//
// Code that is used by or involves more then one storage type.

/*
 * Standard Includes
 */

/*
 * Project Includes
 */

#include "data/data.h"

#include "storage.h"

/*
 * Macros
 */

/*
 * Global Variables
 */

const char* const STYPE_NAMES[NUM_STYPES] = {
	"dense",
	"list",
	"yale"
};

/*
 * Forward Declarations
 */

// FIXME: Template
//static void dense_storage_cast_copy_list_contents(void* lhs, const LIST* rhs, void* default_val, dtype_t l_dtype, dtype_t r_dtype,
//	size_t* pos, const size_t* shape, size_t rank, size_t max_elements, size_t recursions);

// FIXME: Template
//static void dense_storage_cast_copy_list_default(void* lhs, void* default_val, dtype_t l_dtype, dtype_t r_dtype,
//	size_t* pos, const size_t* shape, size_t rank, size_t max_elements, size_t recursions);

/*
 * Functions
 */

/////////////////////////
// Templated Functions //
/////////////////////////

#if false

// To be fixed once each storage class is compiling on its own.

/*
 * Convert (by creating a copy) from list storage to dense storage.
 *
 * FIXME: Add templating.
 */
DENSE_STORAGE* dense_storage_from_list(const LIST_STORAGE* rhs, dtype_t l_dtype) {
  DENSE_STORAGE* lhs;

  // Position in lhs->elements.
  size_t pos = 0;

  // allocate and copy shape
  size_t* shape = ALLOC_N(size_t, rhs->rank);
  memcpy(shape, rhs->shape, rhs->rank * sizeof(size_t));

  lhs = dense_storage_create(l_dtype, shape, rhs->rank, NULL, 0);

  // recursively copy the contents
  dense_storage_cast_copy_list_contents(lhs->elements, rhs->rows, rhs->default_val, l_dtype, rhs->dtype, &pos, shape, lhs->rank, count_storage_max_elements((STORAGE*)rhs), rhs->rank-1);

  return lhs;
}

/*
 * Documentation goes here.
 *
 * FIXME: Add templating.
 */
DENSE_STORAGE* dense_storage_from_yale(const YALE_STORAGE* rhs, dtype_t l_dtype) {
  DENSE_STORAGE* lhs;
  // Position in lhs->elements.
  y_size_t i, j

  // Position in rhs->elements.
  y_size_t ija, ija_next, jj;

  // Position in dense to write to.
  y_size_t pos = 0;

  // Determine zero representation.
  void* R_ZERO = (char*)(rhs->a) + rhs->shape[0] * nm_sizeof[rhs->dtype];

  // Allocate and set shape.
  size_t* shape = ALLOC_N(size_t, rhs->rank);
  memcpy(shape, rhs->shape, rhs->rank * sizeof(size_t));

  lhs = dense_storage_create(l_dtype, shape, rhs->rank, NULL, 0);

  // Walk through rows. For each entry we set in dense, increment pos.
  for (i = rhs->shape[0]; i-- > 0;) {

    // Get boundaries of this row, store in ija and ija_next.
    YaleGetIJA(ija,      rhs, i);
    YaleGetIJA(ija_next, rhs, i+1);

    if (ija == ija_next) {
    	// Row is empty?

			// Write zeros in each column.
			for (j = rhs->shape[1]; j-- > 0;) {

        // Fill in zeros (except for diagonal)
        if (i == j) {
        	cast_copy_value_single((char*)(lhs->elements) + pos*nm_sizeof[l_dtype], (char*)(rhs->a) + i*nm_sizeof[rhs->dtype], l_dtype, rhs->dtype);

				} else {
					cast_copy_value_single((char*)(lhs->elements) + pos*nm_sizeof[l_dtype], R_ZERO, l_dtype, rhs->dtype);
				}

				// Move to next dense position.
        ++pos;
      }

    } else {
      // Row contains entries: write those in each column, interspersed with zeros.
      YaleGetIJA(jj, rhs, ija);

			for (j = rhs->shape[1]; j-- > 0;) {
        if (i == j) {
          cast_copy_value_single((char*)(lhs->elements) + pos*nm_sizeof[l_dtype], (char*)(rhs->a) + i*nm_sizeof[rhs->dtype], l_dtype, rhs->dtype);

        } else if (j == jj) {

          // Copy from rhs.
          cast_copy_value_single((char*)(lhs->elements) + pos*nm_sizeof[l_dtype], (char*)(rhs->a) + ija*nm_sizeof[rhs->dtype], l_dtype, rhs->dtype);

          // Get next.
          ++ija;

          // Increment to next column ID (or go off the end).
          if (ija < ija_next) {
          	YaleGetIJA(jj, rhs, ija);

          } else {
          	jj = rhs->shape[1];
          }

        } else {
        	// j < jj

          // Insert zero.
          cast_copy_value_single((char*)(lhs->elements) + pos*nm_sizeof[l_dtype], R_ZERO, l_dtype, rhs->dtype);
        }

        // Move to next dense position.
        ++pos;
      }
    }
  }

  return lhs;
}

/*
 * Documentation goes here.
 *
 * FIXME: Add templating.
 */
LIST_STORAGE* list_storage_from_dense(const DENSE_STORAGE* rhs, int8_t l_dtype) {
  LIST_STORAGE* lhs;
  size_t pos = 0;
  void* l_default_val = ALLOC_N(char, nm_sizeof[l_dtype]);
  void* r_default_val = ALLOCA_N(char, nm_sizeof[rhs->dtype]); // clean up when finished with this function

  // allocate and copy shape and coords
  size_t *shape = ALLOC_N(size_t, rhs->rank), *coords = ALLOC_N(size_t, rhs->rank);
  memcpy(shape, rhs->shape, rhs->rank * sizeof(size_t));
  memset(coords, 0, rhs->rank * sizeof(size_t));

  // set list default_val to 0
  if (l_dtype == NM_ROBJ) {
  	*(VALUE*)l_default_val = INT2FIX(0);

  } else {
  	memset(l_default_val, 0, nm_sizeof[l_dtype]);
  }

  // need test default value for comparing to elements in dense matrix
  if (rhs->dtype == l_dtype) {
  	r_default_val = l_default_val;

  } else if (rhs->dtype == NM_ROBJ) {
  	*(VALUE*)r_default_val = INT2FIX(0);

  } else {
  	memset(r_default_val, 0, nm_sizeof[rhs->dtype]);
  }

  lhs = create_list_storage(l_dtype, shape, rhs->rank, l_default_val);

  lhs->rows = create_list();
  cast_copy_list_contents_dense(lhs->rows, rhs->elements, r_default_val, l_dtype, rhs->dtype, &pos, coords, rhs->shape, rhs->rank, rhs->rank - 1);

  return lhs;
}

/*
 * Documentation goes here.
 *
 * FIXME: Add templating.
 */
LIST_STORAGE* list_storage_from_yale(const YALE_STORAGE* rhs, int8_t l_dtype) {
  LIST_STORAGE* lhs;
  NODE *last_added, *last_row_added = NULL;
  LIST* curr_row;
  y_size_t ija, ija_next, i, jj;
  bool add_diag;
  void* default_val = ALLOC_N(char, nm_sizeof[l_dtype]);
  void* R_ZERO = (char*)(rhs->a) + rhs->shape[0]*nm_sizeof[rhs->dtype];
  void* insert_val;

  // allocate and copy shape
  size_t *shape = ALLOC_N(size_t, rhs->rank);
  shape[0] = rhs->shape[0]; shape[1] = rhs->shape[1];

  // copy default value from the zero location in the Yale matrix
  SetFuncs[l_dtype][rhs->dtype](1, default_val, 0, R_ZERO, 0);

  lhs = create_list_storage(l_dtype, shape, rhs->rank, default_val);

  if (rhs->rank != 2) {
    rb_raise(nm_eStorageTypeError, "Can only convert matrices of rank 2 from yale.");
  }

  // Walk through rows and columns as if RHS were a dense matrix
  for (i = rhs->shape[0]; i-- > 0;) {

    // Get boundaries of beginning and end of row
    YaleGetIJA(ija, rhs, i);
    YaleGetIJA(ija_next, rhs, i+1);

    // Are we going to need to add a diagonal for this row?
    if (ElemEqEq[rhs->dtype][0]((char*)(rhs->a) + i*nm_sizeof[rhs->dtype], R_ZERO, 1, nm_sizeof[rhs->dtype])) {
    	// zero
    	add_diag = false;

    } else {
    	// nonzero diagonal
    	add_diag = true;
    }

    if (ija < ija_next || add_diag) {

      curr_row = create_list();
      last_added = NULL;

      while (ija < ija_next) {
        YaleGetIJA(jj, rhs, ija); // what column number is this?

        // Is there a nonzero diagonal item between the previously added item and the current one?
        if (jj > i && add_diag) {
          // Allocate and copy insertion value
          insert_val = ALLOC_N(char, nm_sizeof[l_dtype]);
          SetFuncs[l_dtype][rhs->dtype](1, insert_val, 0, (char*)(rhs->a) + i*nm_sizeof[rhs->dtype], 0);

          // insert the item in the list at the appropriate location
          if (last_added) {
          	last_added = list_insert_after(last_added, i, insert_val);

          } else {
          	last_added = list_insert(curr_row, false, i, insert_val);
          }

					// don't add again!
          add_diag = false;
        }

        // now allocate and add the current item
        insert_val = ALLOC_N(char, nm_sizeof[l_dtype]);
        SetFuncs[l_dtype][rhs->dtype](1, insert_val, 0, (char*)(rhs->a) + ija*nm_sizeof[rhs->dtype], 0);

        if (last_added) {
        	last_added = list_insert_after(last_added, jj, insert_val);

        } else {
        	last_added = list_insert(curr_row, false, jj, insert_val);
        }

        ++ija; // move to next entry in Yale matrix
      }

      if (add_diag) {
      	// still haven't added the diagonal.

        insert_val = ALLOC_N(char, nm_sizeof[l_dtype]);
        SetFuncs[l_dtype][rhs->dtype](1, insert_val, 0, (char*)(rhs->a) + i*nm_sizeof[rhs->dtype], 0);

        // insert the item in the list at the appropriate location
        if (last_added) {
        	last_added = list_insert_after(last_added, i, insert_val);

        } else {
        	last_added = list_insert(curr_row, false, i, insert_val);
        }
      }

      // Now add the list at the appropriate location
      if (last_row_added) {
      	last_row_added = list_insert_after(last_row_added, i, curr_row);

      } else {
      	last_row_added = list_insert(lhs->rows, false, i, curr_row);
      }
    }

		// end of walk through rows
  }

  return lhs;
}

/*
 * Documentation goes here.
 *
 * FIXME: Add templating.
 */
YALE_STORAGE* yale_storage_from_dense(const DENSE_STORAGE* rhs, int8_t l_dtype) {
  YALE_STORAGE* lhs;
  size_t i, j;
  y_size_t pos = 0, ndnz = 0, ija;
  size_t* shape;

  // Figure out values to write for zero in yale and compare to zero in dense
  void *R_ZERO = ALLOCA_N(char, nm_sizeof[rhs->dtype]);
  if (rhs->dtype == NM_ROBJ) {
  	*(VALUE*)R_ZERO = INT2FIX(0);

  } else {
  	memset(R_ZERO, 0, nm_sizeof[rhs->dtype]);
  }

  if (rhs->rank != 2) {
    rb_raise(nm_eStorageTypeError, "can only convert matrices of rank 2 to yale");
  }

  // First, count the non-diagonal nonzeros
	for (i = rhs->shape[0]; i-- > 0;) {
		for (j = rhs->shape[1]; j-- > 0;) {
      if (i != j && !ElemEqEq[rhs->dtype][0]((char*)(rhs->elements) + pos*nm_sizeof[rhs->dtype], R_ZERO, 1, nm_sizeof[rhs->dtype])) {
      	++ndnz;
      }

      // move forward 1 position in dense matrix elements array
      ++pos;
    }
  }

  // Copy shape for yale construction
  shape = ALLOC_N(size_t, 2);
  shape[0] = rhs->shape[0];
  shape[1] = rhs->shape[1];

  // Create with minimum possible capacity -- just enough to hold all of the entries
  lhs = yale_storage_create(l_dtype, shape, 2, shape[0] + ndnz + 1);

  // Set the zero position in the yale matrix
  cast_copy_value_single((char*)(lhs->a) + shape[0]*nm_sizeof[l_dtype], R_ZERO, l_dtype, rhs->dtype);

  // Start just after the zero position.
  ija = lhs->shape[0]+1;
  pos = 0;

  // Copy contents
  for (i = 0; i < rhs->shape[0]; ++i) {
    // indicate the beginning of a row in the IJA array
    YaleSetIJA(i, lhs, ija);

    for (j = 0; j < rhs->shape[1]; ++j) {

      if (i == j) { // copy to diagonal
        cast_copy_value_single((char*)(lhs->a) + i*nm_sizeof[l_dtype], (char*)(rhs->elements) + pos*nm_sizeof[rhs->dtype], l_dtype, rhs->dtype);

      } else if (!ElemEqEq[rhs->dtype][0]((char*)(rhs->elements) + pos*nm_sizeof[rhs->dtype], R_ZERO, 1, nm_sizeof[rhs->dtype])) {      // copy nonzero to LU
        YaleSetIJA(ija, lhs, j); // write column index

        cast_copy_value_single((char*)(lhs->a) + ija*nm_sizeof[l_dtype], (char*)(rhs->elements) + pos*nm_sizeof[rhs->dtype], l_dtype, rhs->dtype);

        ++ija;
      }
      ++pos;
    }
  }
  YaleSetIJA(i, lhs, ija); // indicate the end of the last row

  lhs->ndnz = ndnz;

  return lhs;
}

/*
 * Documentation goes here.
 *
 * FIXME: Add templating.
 */
YALE_STORAGE* yale_storage_from_list(const LIST_STORAGE* rhs, int8_t l_dtype) {
  YALE_STORAGE* lhs;
  size_t* shape;
  NODE *i_curr, *j_curr;
  y_size_t ija;
  size_t ndnz = count_list_storage_nd_elements(rhs);

  if (rhs->rank != 2) {
    rb_raise(nm_eStorageTypeError, "can only convert matrices of rank 2 to yale");
  }

  if ((rhs->dtype == NM_ROBJ && *(VALUE*)(rhs->default_val) == INT2FIX(0)) || strncmp(rhs->default_val, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", nm_sizeof[rhs->dtype])) {
    rb_raise(nm_eStorageTypeError, "list matrix must have default value of 0 to convert to yale");
  }

  // Copy shape for yale construction
  shape = ALLOC_N(size_t, 2);
  shape[0] = rhs->shape[0];
  shape[1] = rhs->shape[1];

  lhs = yale_storage_create(l_dtype, shape, 2, shape[0] + ndnz + 1);
  clear_diagonal_and_zero(lhs); // clear the diagonal and the zero location.

  ija = lhs->shape[0]+1;

  for (i_curr = rhs->rows->first; i_curr; i_curr = i_curr->next) {

    // indicate the beginning of a row in the IJA array
    YaleSetIJA(i_curr->key, lhs, ija);

    for (j_curr = ((LIST*)(i_curr->val))->first; j_curr; j_curr = j_curr->next) {
      if (i_curr->key == j_curr->key) {
        // set diagonal
        SetFuncs[l_dtype][rhs->dtype](1, (char*)(lhs->a) + (i_curr->key)*nm_sizeof[l_dtype], 0, j_curr->val, 0);

      } else {
        // set column value
        YaleSetIJA(ija, lhs, j_curr->key); // write column index

        // set cell value
        SetFuncs[l_dtype][rhs->dtype](1, (char*)(lhs->a) + ija*nm_sizeof[l_dtype], 0, j_curr->val, 0);

        ++ija;
      }
    }

    if (!i_curr->next) {
    	// indicate the end of the last row
    	YaleSetIJA(i_curr->key, lhs, ija);
    }
  }

  lhs->ndnz = ndnz;
  return lhs;
}

//////////////////////
// Helper Functions //
//////////////////////

/*
 * Copy list contents into dense recursively.
 */
static void dense_storage_cast_copy_list_contents(void* lhs, const LIST* rhs, void* default_val, dtype_t l_dtype, dtype_t r_dtype,size_t* pos, const size_t* shape, size_t rank, size_t max_elements, size_t recursions) {

  NODE *curr = rhs->first;
  int last_key = -1;
  size_t i = 0;

	for (i = shape[rank - 1 - recursions]; i-- > 0; ++(*pos) {

    if (!curr || (curr->key > (size_t)(last_key+1))) {
      //fprintf(stderr, "pos = %u, dim = %u, curr->key XX, last_key+1 = %d\t", *pos, shape[rank-1-recursions], last_key+1);

      if (recursions == 0) {
      	cast_copy_value_single((char*)lhs + (*pos)*nm_sizeof[l_dtype], default_val, l_dtype, r_dtype);
    		//fprintf(stderr, "zero\n");

      } else {
     		dense_storage_cast_copy_list_default(lhs, default_val, l_dtype, r_dtype, pos, shape, rank, max_elements, recursions-1);
    		//fprintf(stderr, "column of zeros\n");
			}

      ++last_key;

    } else {
      //fprintf(stderr, "pos = %u, dim = %u, curr->key = %u, last_key+1 = %d\t", *pos, shape[rank-1-recursions], curr->key, last_key+1);

      if (recursions == 0) {
      	cast_copy_value_single((char*)lhs + (*pos)*nm_sizeof[l_dtype], curr->val, l_dtype, r_dtype);
    		//fprintf(stderr, "zero\n");

      } else {
      	dense_storage_cast_copy_list_default(lhs, curr->val, default_val, l_dtype, r_dtype, pos, shape, rank, max_elements, recursions-1);
    		//fprintf(stderr, "column of zeros\n");
      }

      last_key = curr->key;
      curr     = curr->next;
    }
  }

  --(*pos);
}

/*
 * Copy a set of default values into dense.
 */
static void dense_storage_cast_copy_list_default(void* lhs, void* default_val, dtype_t l_dtype, dtype_t r_dtype, size_t* pos, const size_t* shape, size_t rank, size_t max_elements, size_t recursions) {
  size_t i;

	for (i = shape[rank - 1 - recursions]; i-- > 0; ++(*pos)) {
    //fprintf(stderr, "default: pos = %u, dim = %u\t", *pos, shape[rank-1-recursions]);

    if (recursions == 0) {
    	cast_copy_value_single((char*)lhs + (*pos)*nm_sizeof[l_dtype], default_val, l_dtype, r_dtype);
    	//fprintf(stderr, "zero\n");

    } else {
    	dense_storage_cast_copy_list_default(lhs, default_val, l_dtype, r_dtype, pos, shape, rank, max_elements, recursions-1);
    	//fprintf(stderr, "column of zeros\n");
  	}
  }

  --(*pos);
}

#endif

<|endoftext|>


<commit_before>/*
===========================================================================

Daemon GPL Source Code
Copyright (C) 2013 Unvanquished Developers

This file is part of the Daemon GPL Source Code (Daemon Source Code).

Daemon Source Code is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

Daemon Source Code is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with Daemon Source Code.  If not, see <http://www.gnu.org/licenses/>.

===========================================================================
*/

#include "Command.h"
#include "Log.h"

#include "../engine/qcommon/qcommon.h"
#include "../engine/framework/CommandSystem.h"
#include "../engine/framework/CvarSystem.h"

namespace Cmd {

    std::string Escape(Str::StringRef text) {
        std::string res = "\"";

        for (size_t i = 0; i < text.size(); i ++) {
            if (text[i] == '\\') {
                res.append("\\\\");
            } else if (text[i] == '\"') {
                res.append("\\\"");
            } else {
                res.push_back(text[i]);
            }
        }

        res += "\"";
        return res;
    }

    static bool SkipSpaces(std::string::const_iterator& in, std::string::const_iterator end)
    {
        bool foundSpace = false;
        while (in != end) {
            // Handle spaces
            if ((in[0] & 0xff) <= ' ') {
                foundSpace = true;
                in++;
            } else if (in + 1 != end) {
                // Handle // comments
                if (in[0] == '/' && in[1] == '/') {
                    in = end;
                    return true;
                }
                // Handle /* */ comments
                if (in[0] == '/' && in[1] == '*') {
                    foundSpace = true;
                    in += 2;
                    while (in != end) {
                        if (in + 1 != end && in[0] == '*' && in[1] == '/') {
                            in += 2;
                            break;
                        } else {
                            in++;
                        }
                    }
                } else {
                    // Non-space character
                    return foundSpace;
                }
            } else {
                // Non-space character
                return foundSpace;
            }
        }
        return true;
    }

    const char* SplitCommand(const char* start, const char* end) {
        bool inQuote = false;
        bool inComment = false;
        bool inInlineComment = false;

        while (start != end) {
            //End of comment
            if (inInlineComment && start + 1 != end && start[0] == '*' && start[1] == '/') {
                inInlineComment = false;
                start += 2;
                continue;
            }

            //Ignore everything else in an inline comment
            if (inInlineComment) {
                start++;
                continue;
            }

            //Start of comment
            if (not inQuote && not inComment && start + 1 != end && start[0] == '/' && start[1] == '*') {
                inInlineComment = true;
                start += 2;
                continue;
            }
            if (not inQuote && start + 1 != end && start[0] == '/' && start[1] == '/') {
                inComment = true;
                start += 2;
                continue;
            }

            //Escaped quote
            if (inQuote && start + 1 != end && start[0] == '\\') {
                start += 2;
                continue;
            }

            //Quote
            if (start[0] == '"') {
                inQuote = !inQuote;
                start++;
                continue;
            }

            //Semicolon
            if (start[0] == ';' && !inQuote && !inComment) {
                return start + 1;
            }

            //Newline
            if (start[0] == '\n') {
                return start + 1;
            }

            //Normal character
            start++;
        }

        return end;
    }

    std::string SubstituteCvars(Str::StringRef text) {
        const char* raw_text = text.data();
        std::string result;

        bool isEscaped = false;
        bool inCvarName = false;
        int lastBlockStart = 0;

        //Cvar are delimited by $ so we parse a bloc at a time
        for(size_t i = 0; i < text.size(); i++){

            // a \ escapes the next letter so we don't use it for block delimitation
            if (isEscaped) {
                isEscaped = false;
                continue;
            }

            if (text[i] == '\\') {
                isEscaped = true;

            } else if (text[i] == '$') {
                //Found a block, every second block is a cvar name block
                std::string block(raw_text + lastBlockStart, i - lastBlockStart);

                if (inCvarName) {
                    result += Escape(Cvar::GetValue(block));
                    inCvarName = false;

                } else {
                    result += block;
                    inCvarName = true;
                }

                lastBlockStart = i + 1;
            }
        }

        //Handle the last block
        if (inCvarName) {
            Com_Printf("Warning: last CVar substitution block not closed in %s\n", raw_text);
        } else {
            std::string block(raw_text + lastBlockStart, text.size() - lastBlockStart);
            result += block;
        }

        return result;
    }

    /*
    ===============================================================================

    Cmd::CmdArgs

    ===============================================================================
    */

    Args::Args() {
    }

    Args::Args(std::vector<std::string> args_) {
        args = std::move(args_);

        for (size_t i = 0; i < args.size(); i++) {
            argsStarts.push_back(cmd.size());
            cmd += Escape(args[i]);
            if (i != args.size() - 1) {
                cmd += " ";
            }
        }
    }

    Args::Args(std::string command) {
        cmd = std::move(command);

        std::string::const_iterator in = cmd.begin();

        //Skip leading whitespace
        SkipSpaces(in, cmd.end());

        //Return if the cmd is empty
        if (in == cmd.end())
            return;

        //Build arg tokens
        std::string currentToken;
        int currentTokenStart = in - cmd.begin();
        while (true) {
            //Check for end of current token
            if (SkipSpaces(in, cmd.end())) {
                 args.push_back(std::move(currentToken));
                 argsStarts.push_back(currentTokenStart);
                 if (in == cmd.end())
                    return;
                 currentToken.clear();
                 currentTokenStart = in - cmd.begin();
            }

            //Handle quoted strings
            if (in[0] == '\"') {
                in++;
                //Read all characters until quote end
                while (in != cmd.end()) {
                    if (in[0] == '\"') {
                        in++;
                        break;
                    } else if (in + 1 !=  cmd.end() && in[0] == '\\') {
                        currentToken.push_back(in[1]);
                        in += 2;
                    } else {
                        currentToken.push_back(in[0]);
                        in++;
                    }
                }
            } else {
                //Handle normal character
                currentToken.push_back(in[0]);
                in++;
            }
        }
    }

    int Args::Argc() const {
        return args.size();
    }

    const std::string& Args::Argv(int argNum) const {
        return args[argNum];
    }

    std::string Args::EscapedArgs(int start, int end) const {
        std::string res;

        if (end < 0) {
            end = args.size() - 1;
        }

        for (int i = start; i < end + 1; i++) {
            if (i != start) {
                res += " ";
            }
            res += Escape(args[i]);
        }

        return res;
    }

    const std::string& Args::RawCmd() const {
        return cmd;
    }

    std::string Args::RawArgsFrom(unsigned start) const {
        if (start < argsStarts.size()) {
            return std::string(cmd.begin() + argsStarts[start], cmd.end());
        } else {
            return "";
        }
    }

    int Args::PosToArg(int pos) const {
        for (int i = argsStarts.size(); i-->0;) {
            if (argsStarts[i] <= pos) {
                return i;
            }
        }

        return -1;
    }

    int Args::ArgStartPos(unsigned argNum) const {
        if (argNum > argsStarts.size()) {
            return cmd.size();
        }
        return argsStarts[argNum];
    }

    std::string Args::ArgPrefix(int pos) const {
        int argNum = PosToArg(pos);
        return args[argNum].substr(0, pos);
    }

    const std::vector<std::string>& Args::ArgVector() const {
        return args;
    }

    int Args::size() const {
        return Argc();
    }

    const std::string& Args::operator[] (int argNum) const {
        return Argv(argNum);
    }

    /*
    ===============================================================================

    Cmd::CmdBase

    ===============================================================================
    */

    CmdBase::CmdBase(const int flags): flags(flags) {
    }

    CompletionResult CmdBase::Complete(int argNum, const Args& args, const std::string& prefix) const {
        Q_UNUSED(argNum);
        Q_UNUSED(args);
        Q_UNUSED(prefix);
        return {};
    }

    void CmdBase::PrintUsage(const Args& args, Str::StringRef syntax, Str::StringRef description) const {
        if(description.empty()) {
            Print("%s: %s %s\n", _("usage"), args.Argv(0).c_str(), syntax.c_str());
        } else {
            Print("%s: %s %s — %s\n", _("usage"), args.Argv(0).c_str(), syntax.c_str(), description.c_str());
        }
    }

    int CmdBase::GetFlags() const {
        return flags;
    }

    void CmdBase::ExecuteAfter(Str::StringRef text, bool parseCvars) const {
        GetEnv().ExecuteAfter(text, parseCvars);
    }

    Environment& CmdBase::GetEnv() const {
        return *Cmd::GetEnv();
    }

    StaticCmd::StaticCmd(std::string name, const int flags, std::string description)
    :CmdBase(flags){
        //Register this command statically
        AddCommand(std::move(name), *this, std::move(description));
    }

}
<commit_msg>Remove extra newlines in PrintUsage<commit_after>/*
===========================================================================

Daemon GPL Source Code
Copyright (C) 2013 Unvanquished Developers

This file is part of the Daemon GPL Source Code (Daemon Source Code).

Daemon Source Code is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

Daemon Source Code is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with Daemon Source Code.  If not, see <http://www.gnu.org/licenses/>.

===========================================================================
*/

#include "Command.h"
#include "Log.h"

#include "../engine/qcommon/qcommon.h"
#include "../engine/framework/CommandSystem.h"
#include "../engine/framework/CvarSystem.h"

namespace Cmd {

    std::string Escape(Str::StringRef text) {
        std::string res = "\"";

        for (size_t i = 0; i < text.size(); i ++) {
            if (text[i] == '\\') {
                res.append("\\\\");
            } else if (text[i] == '\"') {
                res.append("\\\"");
            } else {
                res.push_back(text[i]);
            }
        }

        res += "\"";
        return res;
    }

    static bool SkipSpaces(std::string::const_iterator& in, std::string::const_iterator end)
    {
        bool foundSpace = false;
        while (in != end) {
            // Handle spaces
            if ((in[0] & 0xff) <= ' ') {
                foundSpace = true;
                in++;
            } else if (in + 1 != end) {
                // Handle // comments
                if (in[0] == '/' && in[1] == '/') {
                    in = end;
                    return true;
                }
                // Handle /* */ comments
                if (in[0] == '/' && in[1] == '*') {
                    foundSpace = true;
                    in += 2;
                    while (in != end) {
                        if (in + 1 != end && in[0] == '*' && in[1] == '/') {
                            in += 2;
                            break;
                        } else {
                            in++;
                        }
                    }
                } else {
                    // Non-space character
                    return foundSpace;
                }
            } else {
                // Non-space character
                return foundSpace;
            }
        }
        return true;
    }

    const char* SplitCommand(const char* start, const char* end) {
        bool inQuote = false;
        bool inComment = false;
        bool inInlineComment = false;

        while (start != end) {
            //End of comment
            if (inInlineComment && start + 1 != end && start[0] == '*' && start[1] == '/') {
                inInlineComment = false;
                start += 2;
                continue;
            }

            //Ignore everything else in an inline comment
            if (inInlineComment) {
                start++;
                continue;
            }

            //Start of comment
            if (not inQuote && not inComment && start + 1 != end && start[0] == '/' && start[1] == '*') {
                inInlineComment = true;
                start += 2;
                continue;
            }
            if (not inQuote && start + 1 != end && start[0] == '/' && start[1] == '/') {
                inComment = true;
                start += 2;
                continue;
            }

            //Escaped quote
            if (inQuote && start + 1 != end && start[0] == '\\') {
                start += 2;
                continue;
            }

            //Quote
            if (start[0] == '"') {
                inQuote = !inQuote;
                start++;
                continue;
            }

            //Semicolon
            if (start[0] == ';' && !inQuote && !inComment) {
                return start + 1;
            }

            //Newline
            if (start[0] == '\n') {
                return start + 1;
            }

            //Normal character
            start++;
        }

        return end;
    }

    std::string SubstituteCvars(Str::StringRef text) {
        const char* raw_text = text.data();
        std::string result;

        bool isEscaped = false;
        bool inCvarName = false;
        int lastBlockStart = 0;

        //Cvar are delimited by $ so we parse a bloc at a time
        for(size_t i = 0; i < text.size(); i++){

            // a \ escapes the next letter so we don't use it for block delimitation
            if (isEscaped) {
                isEscaped = false;
                continue;
            }

            if (text[i] == '\\') {
                isEscaped = true;

            } else if (text[i] == '$') {
                //Found a block, every second block is a cvar name block
                std::string block(raw_text + lastBlockStart, i - lastBlockStart);

                if (inCvarName) {
                    result += Escape(Cvar::GetValue(block));
                    inCvarName = false;

                } else {
                    result += block;
                    inCvarName = true;
                }

                lastBlockStart = i + 1;
            }
        }

        //Handle the last block
        if (inCvarName) {
            Com_Printf("Warning: last CVar substitution block not closed in %s\n", raw_text);
        } else {
            std::string block(raw_text + lastBlockStart, text.size() - lastBlockStart);
            result += block;
        }

        return result;
    }

    /*
    ===============================================================================

    Cmd::CmdArgs

    ===============================================================================
    */

    Args::Args() {
    }

    Args::Args(std::vector<std::string> args_) {
        args = std::move(args_);

        for (size_t i = 0; i < args.size(); i++) {
            argsStarts.push_back(cmd.size());
            cmd += Escape(args[i]);
            if (i != args.size() - 1) {
                cmd += " ";
            }
        }
    }

    Args::Args(std::string command) {
        cmd = std::move(command);

        std::string::const_iterator in = cmd.begin();

        //Skip leading whitespace
        SkipSpaces(in, cmd.end());

        //Return if the cmd is empty
        if (in == cmd.end())
            return;

        //Build arg tokens
        std::string currentToken;
        int currentTokenStart = in - cmd.begin();
        while (true) {
            //Check for end of current token
            if (SkipSpaces(in, cmd.end())) {
                 args.push_back(std::move(currentToken));
                 argsStarts.push_back(currentTokenStart);
                 if (in == cmd.end())
                    return;
                 currentToken.clear();
                 currentTokenStart = in - cmd.begin();
            }

            //Handle quoted strings
            if (in[0] == '\"') {
                in++;
                //Read all characters until quote end
                while (in != cmd.end()) {
                    if (in[0] == '\"') {
                        in++;
                        break;
                    } else if (in + 1 !=  cmd.end() && in[0] == '\\') {
                        currentToken.push_back(in[1]);
                        in += 2;
                    } else {
                        currentToken.push_back(in[0]);
                        in++;
                    }
                }
            } else {
                //Handle normal character
                currentToken.push_back(in[0]);
                in++;
            }
        }
    }

    int Args::Argc() const {
        return args.size();
    }

    const std::string& Args::Argv(int argNum) const {
        return args[argNum];
    }

    std::string Args::EscapedArgs(int start, int end) const {
        std::string res;

        if (end < 0) {
            end = args.size() - 1;
        }

        for (int i = start; i < end + 1; i++) {
            if (i != start) {
                res += " ";
            }
            res += Escape(args[i]);
        }

        return res;
    }

    const std::string& Args::RawCmd() const {
        return cmd;
    }

    std::string Args::RawArgsFrom(unsigned start) const {
        if (start < argsStarts.size()) {
            return std::string(cmd.begin() + argsStarts[start], cmd.end());
        } else {
            return "";
        }
    }

    int Args::PosToArg(int pos) const {
        for (int i = argsStarts.size(); i-->0;) {
            if (argsStarts[i] <= pos) {
                return i;
            }
        }

        return -1;
    }

    int Args::ArgStartPos(unsigned argNum) const {
        if (argNum > argsStarts.size()) {
            return cmd.size();
        }
        return argsStarts[argNum];
    }

    std::string Args::ArgPrefix(int pos) const {
        int argNum = PosToArg(pos);
        return args[argNum].substr(0, pos);
    }

    const std::vector<std::string>& Args::ArgVector() const {
        return args;
    }

    int Args::size() const {
        return Argc();
    }

    const std::string& Args::operator[] (int argNum) const {
        return Argv(argNum);
    }

    /*
    ===============================================================================

    Cmd::CmdBase

    ===============================================================================
    */

    CmdBase::CmdBase(const int flags): flags(flags) {
    }

    CompletionResult CmdBase::Complete(int argNum, const Args& args, const std::string& prefix) const {
        Q_UNUSED(argNum);
        Q_UNUSED(args);
        Q_UNUSED(prefix);
        return {};
    }

    void CmdBase::PrintUsage(const Args& args, Str::StringRef syntax, Str::StringRef description) const {
        if(description.empty()) {
            Print("%s: %s %s", _("usage"), args.Argv(0).c_str(), syntax.c_str());
        } else {
            Print("%s: %s %s — %s", _("usage"), args.Argv(0).c_str(), syntax.c_str(), description.c_str());
        }
    }

    int CmdBase::GetFlags() const {
        return flags;
    }

    void CmdBase::ExecuteAfter(Str::StringRef text, bool parseCvars) const {
        GetEnv().ExecuteAfter(text, parseCvars);
    }

    Environment& CmdBase::GetEnv() const {
        return *Cmd::GetEnv();
    }

    StaticCmd::StaticCmd(std::string name, const int flags, std::string description)
    :CmdBase(flags){
        //Register this command statically
        AddCommand(std::move(name), *this, std::move(description));
    }

}
<|endoftext|>


<commit_before>#ifndef COMPOSE_NODE_HPP
#define COMPOSE_NODE_HPP

#include <memory>
#include <map>
#include <set>
#include <stdexcept>
#include <east.hpp>
#include "core/composed_node.hpp"
#include "phi_node.hpp"
#include "const_node.hpp"
#include "sqrt_node.hpp"
#include "log_node.hpp"
#include "log10_node.hpp"
#include "logit_node.hpp"
#include "ilogit_node.hpp"
#include "sin_node.hpp"
#include "cos_node.hpp"
#include "tan_node.hpp"
#include "arithmetic_node.hpp"
#include "phi_node.hpp"
#include <atomic>

namespace mcmc_utilities
{
  template <typename T>
  class str_node
    :public composed_node<T,std::string>
  {
  public:
    static std::map<std::string,std::shared_ptr<abstract_node_factory<T> > > node_factories;
    static void init_node_factories()
    {
      std::atomic<bool> initialized(false);

      if(initialized)
	{
	  return;
	}

      auto& node_factories=str_node::node_factories;

      node_factories["add"]=std::shared_ptr<abstract_node_factory<T> >(new add_node_factory<T>());
      node_factories["sub"]=std::shared_ptr<abstract_node_factory<T> >(new sub_node_factory<T>());
      node_factories["neg"]=std::shared_ptr<abstract_node_factory<T> >(new neg_node_factory<T>());
      node_factories["pos"]=std::shared_ptr<abstract_node_factory<T> >(new pos_node_factory<T>());
      node_factories["mul"]=std::shared_ptr<abstract_node_factory<T> >(new mul_node_factory<T>());
      node_factories["div"]=std::shared_ptr<abstract_node_factory<T> >(new div_node_factory<T>());
      node_factories["pow"]=std::shared_ptr<abstract_node_factory<T> >(new pow_node_factory<T>());
      node_factories["con"]=std::shared_ptr<abstract_node_factory<T> >(new const_node_factory<T>());
      node_factories["phi"]=std::shared_ptr<abstract_node_factory<T> >(new phi_node_factory<T>());
      node_factories["sqrt"]=std::shared_ptr<abstract_node_factory<T> >(new sqrt_node_factory<T>());
      node_factories["log"]=std::shared_ptr<abstract_node_factory<T> >(new log_node_factory<T>());
      node_factories["log10"]=std::shared_ptr<abstract_node_factory<T> >(new log10_node_factory<T>());
      node_factories["sin"]=std::shared_ptr<abstract_node_factory<T> >(new sin_node_factory<T>());
      node_factories["cos"]=std::shared_ptr<abstract_node_factory<T> >(new cos_node_factory<T>());
      node_factories["tan"]=std::shared_ptr<abstract_node_factory<T> >(new tan_node_factory<T>());
      node_factories["logit"]=std::shared_ptr<abstract_node_factory<T> >(new logit_node_factory<T>());
      node_factories["ilogit"]=std::shared_ptr<abstract_node_factory<T> >(new ilogit_node_factory<T>());

      initialized=true;
    }
  private:
    void count_input(const east::expression_node& en,
		std::set<std::string>& tags)
    {
      for(int i=0;i<en.get_num_of_parents();++i)
	{
	  count_input(en.get_parent(i),tags);
	}
      if(en.get_kind()=="var")
	{
	  tags.insert(en.get_symbol());
	}
    }

  private:
    std::shared_ptr<deterministic_node<T> > add_node(const east::expression_node& en,int& n)
    {
      if(en.get_kind()=="con")
	{
	  if(this->elements.count(en.get_symbol())==0)
	    {
	      auto p=std::dynamic_pointer_cast<deterministic_node<T> >(node_factories["con"]->get_node({(T)std::stod(en.get_symbol())}));
	      this->elements[en.get_symbol()]=p;
	      return p;
	    }
	  else
	    {
	      return this->elements[en.get_symbol()];
	    }
	}
      else if(en.get_kind()=="var")
	{
	  return this->elements[en.get_symbol()];
	}
      else
	{
	  std::shared_ptr<deterministic_node<T> > p;
	  if(en.get_kind()!="ftn")
	    {
	      p=std::dynamic_pointer_cast<deterministic_node<T> >(node_factories[en.get_kind()]->get_node());
	    }
	  else
	    {
	      p=std::dynamic_pointer_cast<deterministic_node<T> >(node_factories[en.get_symbol()]->get_node());
	    }
	  for(int i=0;i<en.get_num_of_parents();++i)
	    {
	      auto p1=add_node(en.get_parent(i),n);
	      p->connect_to_parent(p1.get(),i,0);
	    }
	  std::string tag="node_"+std::to_string(n++);
	  this->elements[tag]=p;
	  return p;
	}
    }

  public:
    str_node(const std::string& expression,
		    const std::vector<std::string>& input_names)
      :composed_node<T,std::string>(input_names.size(),1)
    {
      str_node::init_node_factories();
      east::parser parser;
      std::shared_ptr<east::expression_node> enode=parser.parse(expression);
      std::set<std::string> tags;

      count_input(*enode,tags);
      if(input_names.size()!=tags.size())
	{
	  throw std::logic_error("input names mismatch");
	}
      for(auto s:input_names)
	{
	  if(tags.count(s)!=1)
	    {
	      throw std::logic_error("input names mismatch");
	    }
	}
      for(int i=0;i<input_names.size();++i)
	{
	  std::shared_ptr<deterministic_node<T> > pn(new forward_node<T>);
	  this->elements.insert(std::make_pair(input_names[i],pn));
	  this->param_list.push_back(pn);
	}
      int n=1;
      auto pn=add_node(*enode,n);
      this->return_list.push_back({pn,0});
    }
  };

  template <typename T>
  std::map<std::string,std::shared_ptr<abstract_node_factory<T> > > str_node<T>::node_factories;

  template <typename T>
  void add_node_to(composed_node<T,std::string>& cn,const east::expression_node& en,std::set<std::string>& tag_set)
  {

  }
};

#endif
<commit_msg>	修改：     ../nodes/str_node.hpp<commit_after>#ifndef COMPOSE_NODE_HPP
#define COMPOSE_NODE_HPP

#include <memory>
#include <map>
#include <set>
#include <stdexcept>
#include <east.hpp>
#include "core/composed_node.hpp"
#include "phi_node.hpp"
#include "const_node.hpp"
#include "sqrt_node.hpp"
#include "log_node.hpp"
#include "log10_node.hpp"
#include "logit_node.hpp"
#include "ilogit_node.hpp"
#include "sin_node.hpp"
#include "cos_node.hpp"
#include "tan_node.hpp"
#include "arithmetic_node.hpp"
#include "phi_node.hpp"
#include <atomic>

namespace mcmc_utilities
{
  template <typename T>
  class str_node
    :public composed_node<T,std::string>
  {
  public:
    static std::map<std::string,std::shared_ptr<abstract_node_factory<T> > > node_factories;
    static void init_node_factories()
    {
      std::atomic<bool> initialized(false);

      if(initialized)
	{
	  return;
	}

      auto& node_factories=str_node::node_factories;

      register_function({"add"},std::shared_ptr<abstract_node_factory<T> >(new add_node_factory<T>()));
      register_function({"sub"},std::shared_ptr<abstract_node_factory<T> >(new sub_node_factory<T>()));
      register_function({"neg"},std::shared_ptr<abstract_node_factory<T> >(new neg_node_factory<T>()));
      register_function({"pos"},std::shared_ptr<abstract_node_factory<T> >(new pos_node_factory<T>()));
      register_function({"mul"},std::shared_ptr<abstract_node_factory<T> >(new mul_node_factory<T>()));
      register_function({"div"},std::shared_ptr<abstract_node_factory<T> >(new div_node_factory<T>()));
      register_function({"pow"},std::shared_ptr<abstract_node_factory<T> >(new pow_node_factory<T>()));
      register_function({"con"},std::shared_ptr<abstract_node_factory<T> >(new const_node_factory<T>()));
      register_function({"phi"},std::shared_ptr<abstract_node_factory<T> >(new phi_node_factory<T>()));
      register_function({"sqrt"},std::shared_ptr<abstract_node_factory<T> >(new sqrt_node_factory<T>()));
      register_function({"log"},std::shared_ptr<abstract_node_factory<T> >(new log_node_factory<T>()));
      register_function({"log10"},std::shared_ptr<abstract_node_factory<T> >(new log10_node_factory<T>()));
      register_function({"sin"},std::shared_ptr<abstract_node_factory<T> >(new sin_node_factory<T>()));
      register_function({"cos"},std::shared_ptr<abstract_node_factory<T> >(new cos_node_factory<T>()));
      register_function({"tan"},std::shared_ptr<abstract_node_factory<T> >(new tan_node_factory<T>()));
      register_function({"logit"},std::shared_ptr<abstract_node_factory<T> >(new logit_node_factory<T>()));
      register_function({"ilogit"},std::shared_ptr<abstract_node_factory<T> >(new ilogit_node_factory<T>()));

      initialized=true;
    }

    static void register_function(const std::string& name,const std::shared_ptr<abstract_node_factory<T> >& ptr)
    {
      node_factories[name]=ptr;
    }

    static void unregister_function(const std::string& name)
    {
      node_factories.erase(name);
    }

  private:
    void count_input(const east::expression_node& en,
		std::set<std::string>& tags)
    {
      for(int i=0;i<en.get_num_of_parents();++i)
	{
	  count_input(en.get_parent(i),tags);
	}
      if(en.get_kind()=="var")
	{
	  tags.insert(en.get_symbol());
	}
    }

  private:
    std::shared_ptr<deterministic_node<T> > add_node(const east::expression_node& en,int& n)
    {
      if(en.get_kind()=="con")
	{
	  if(this->elements.count(en.get_symbol())==0)
	    {
	      auto p=std::dynamic_pointer_cast<deterministic_node<T> >(node_factories["con"]->get_node({(T)std::stod(en.get_symbol())}));
	      this->elements[en.get_symbol()]=p;
	      return p;
	    }
	  else
	    {
	      return this->elements[en.get_symbol()];
	    }
	}
      else if(en.get_kind()=="var")
	{
	  return this->elements[en.get_symbol()];
	}
      else
	{
	  std::shared_ptr<deterministic_node<T> > p;
	  if(en.get_kind()!="ftn")
	    {
	      p=std::dynamic_pointer_cast<deterministic_node<T> >(node_factories[en.get_kind()]->get_node());
	    }
	  else
	    {
	      p=std::dynamic_pointer_cast<deterministic_node<T> >(node_factories[en.get_symbol()]->get_node());
	    }
	  for(int i=0;i<en.get_num_of_parents();++i)
	    {
	      auto p1=add_node(en.get_parent(i),n);
	      p->connect_to_parent(p1.get(),i,0);
	    }
	  std::string tag="node_"+std::to_string(n++);
	  this->elements[tag]=p;
	  return p;
	}
    }

  public:
    str_node(const std::string& expression,
		    const std::vector<std::string>& input_names)
      :composed_node<T,std::string>(input_names.size(),1)
    {
      str_node::init_node_factories();
      east::parser parser;
      std::shared_ptr<east::expression_node> enode=parser.parse(expression);
      std::set<std::string> tags;

      count_input(*enode,tags);
      if(input_names.size()!=tags.size())
	{
	  throw std::logic_error("input names mismatch");
	}
      for(auto s:input_names)
	{
	  if(tags.count(s)!=1)
	    {
	      throw std::logic_error("input names mismatch");
	    }
	}
      for(int i=0;i<input_names.size();++i)
	{
	  std::shared_ptr<deterministic_node<T> > pn(new forward_node<T>);
	  this->elements.insert(std::make_pair(input_names[i],pn));
	  this->param_list.push_back(pn);
	}
      int n=1;
      auto pn=add_node(*enode,n);
      this->return_list.push_back({pn,0});
    }
  };

  template <typename T>
  std::map<std::string,std::shared_ptr<abstract_node_factory<T> > > str_node<T>::node_factories;

  template <typename T>
  void add_node_to(composed_node<T,std::string>& cn,const east::expression_node& en,std::set<std::string>& tag_set)
  {

  }
};

#endif
<|endoftext|>


<commit_before>#include <immintrin.h>
#include <cstdint>
#include <cassert>

char* remove_spaces__avx512vbmi(const char* src, char* dst, size_t n) {

    assert(n % 64 == 0);

    // values 0..63
    const __m512i no_gaps_indices = _mm512_setr_epi32(
        0x03020100, 0x07060504, 0x0b0a0908, 0x0f0e0d0c,
        0x13121110, 0x17161514, 0x1b1a1918, 0x1f1e1d1c,
        0x23222120, 0x27262524, 0x2b2a2928, 0x2f2e2d2c,
        0x33323130, 0x37363534, 0x3b3a3938, 0x3f3e3d3c
    );

    const __m512i ones   = _mm512_set1_epi8(1);
    const __m512i NL     = _mm512_set1_epi8('\n');
    const __m512i CR     = _mm512_set1_epi8('\r');
    const __m512i spaces = _mm512_set1_epi8(' ');

    size_t len;
    for (size_t i=0; i < n; i += 64) {

        const __m512i input  = _mm512_loadu_si512((const __m512i*)(src + i));
        __m512i output;

        uint64_t mask = _mm512_cmpeq_epi8_mask(input, spaces)
                      | _mm512_cmpeq_epi8_mask(input, NL)
                      | _mm512_cmpeq_epi8_mask(input, CR);

        if (mask) {
            len = 64 - __builtin_popcountll(mask);
            __m512i indices = no_gaps_indices;
            __m512i increment = ones;

            uint64_t first;
            uint64_t prev;

            first = (mask & -mask); // isolate the first bit set
            prev  = first;
            mask ^= first;          // reset it in the mask
            mask >>= 1;             // and shift by 1

            while (mask) {
                const uint64_t curr = (mask & -mask);
                mask ^= curr;
                mask >>= 1;

                if (prev == curr) {
                    // run continues
                    increment = _mm512_add_epi8(increment, ones);
                    prev = curr;
                } else {
                    // new run
                    indices = _mm512_mask_add_epi8(indices, ~(first - 1), indices, increment);

                    first = curr;
                    prev  = curr;
                    increment = ones;
                }
            }

            indices = _mm512_mask_add_epi8(indices, ~(first - 1), indices, increment);

            output = _mm512_permutexvar_epi8(indices, input);
        } else {
            output = input;
            len = 64;
        }

        _mm512_storeu_si512((__m512i*)(dst), output);
        dst += len;
    }

    return dst;
}

<commit_msg>Use PEXT to build up the shuffle indices in parallel<commit_after>#include <immintrin.h>
#include <cstdint>
#include <cassert>

char* remove_spaces__avx512vbmi(const char* src, char* dst, size_t n) {
    assert(n % 64 == 0);

    const __m512i NL     = _mm512_set1_epi8('\n');
    const __m512i CR     = _mm512_set1_epi8('\r');
    const __m512i spaces = _mm512_set1_epi8(' ');

    uint64_t index_masks[6] = {
        0xaaaaaaaaaaaaaaaa,
        0xcccccccccccccccc,
        0xf0f0f0f0f0f0f0f0,
        0xff00ff00ff00ff00,
        0xffff0000ffff0000,
        0xffffffff00000000,
    };
    const __m512i bit_masks[6] = {
        _mm512_set1_epi8(1),
        _mm512_set1_epi8(2),
        _mm512_set1_epi8(4),
        _mm512_set1_epi8(8),
        _mm512_set1_epi8(16),
        _mm512_set1_epi8(32),
    };

    size_t len;
    for (size_t i=0; i < n; i += 64) {
        const __m512i input  = _mm512_loadu_si512((const __m512i*)(src + i));
        __m512i output;

        uint64_t mask = _mm512_cmpeq_epi8_mask(input, spaces)
                      | _mm512_cmpeq_epi8_mask(input, NL)
                      | _mm512_cmpeq_epi8_mask(input, CR);

        if (mask) {
            mask = ~mask;
            __m512i indices = _mm512_set1_epi8(0);
            for (size_t index = 0; index < 6; index++) {
                uint64_t m = _pext_u64(index_masks[index], mask);
                indices = _mm512_mask_add_epi8(indices, m, indices, bit_masks[index]);
            }

            output = _mm512_permutexvar_epi8(indices, input);
        } else {
            output = input;
            len = 64;
        }

        _mm512_storeu_si512((__m512i*)(dst), output);
        dst += len;
    }

    return dst;
}
<|endoftext|>


<commit_before>/**
 * Copyright 2016 Yahoo Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "Url.h"
#include <gtest/gtest.h>

using namespace pulsar;

TEST(UrlTest, testUrl) {
    Url url;

    ASSERT_TRUE(Url::parse("http://example.com", url));
    ASSERT_EQ("http", url.protocol());
    ASSERT_EQ(80, url.port());

    ASSERT_TRUE(Url::parse("https://example.com", url));
    ASSERT_EQ("https", url.protocol());
    ASSERT_EQ(443, url.port());

    ASSERT_TRUE(Url::parse("http://example.com:8080", url));
    ASSERT_EQ("http", url.protocol());
    ASSERT_EQ(8080, url.port());

    ASSERT_TRUE(Url::parse("http://example.com:8080/", url));
    ASSERT_EQ("http", url.protocol());
    ASSERT_EQ(8080, url.port());

    ASSERT_TRUE(Url::parse("http://example.com", url));
    ASSERT_EQ("http", url.protocol());
    ASSERT_EQ(80, url.port());

    ASSERT_TRUE(Url::parse("http://example.com:8080/test/my/path", url));
    ASSERT_EQ("http", url.protocol());
    ASSERT_EQ(8080, url.port());

    ASSERT_TRUE(Url::parse("http://example.com:8080/test/my/path?key=value#adsasda", url));
    ASSERT_EQ("http", url.protocol());
    ASSERT_EQ(8080, url.port());

    ASSERT_TRUE(Url::parse("pulsar://example.com:8080", url));
    ASSERT_EQ("pulsar", url.protocol());
    ASSERT_EQ(8080, url.port());

    ASSERT_TRUE(Url::parse("pulsar://example.com", url));
    ASSERT_EQ("pulsar", url.protocol());
    ASSERT_EQ(6650, url.port());
}
<commit_msg>Added more checks in CPP Client URL Tests (#312)<commit_after>/**
 * Copyright 2016 Yahoo Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "Url.h"
#include <gtest/gtest.h>

using namespace pulsar;

TEST(UrlTest, testUrl) {
    Url url;

    ASSERT_TRUE(Url::parse("http://example.com", url));
    ASSERT_EQ("http", url.protocol());
    ASSERT_EQ(80, url.port());

    ASSERT_TRUE(Url::parse("https://example.com", url));
    ASSERT_EQ("https", url.protocol());
    ASSERT_EQ(443, url.port());

    ASSERT_TRUE(Url::parse("http://example.com:8080", url));
    ASSERT_EQ("http", url.protocol());
    ASSERT_EQ(8080, url.port());

    ASSERT_TRUE(Url::parse("http://example.com:8080/", url));
    ASSERT_EQ("http", url.protocol());
    ASSERT_EQ(8080, url.port());

    ASSERT_TRUE(Url::parse("http://example.com", url));
    ASSERT_EQ("example.com", url.host());
    ASSERT_EQ("http", url.protocol());
    ASSERT_EQ(80, url.port());

    ASSERT_TRUE(Url::parse("http://example.com:8080/test/my/path", url));
    ASSERT_EQ("example.com", url.host());
    ASSERT_EQ("http", url.protocol());
    ASSERT_EQ(8080, url.port());

    ASSERT_TRUE(Url::parse("http://example.com:8080/test/my/path?key=value#adsasda", url));
    ASSERT_EQ("example.com", url.host());
    ASSERT_EQ("http", url.protocol());
    ASSERT_EQ(8080, url.port());

    ASSERT_TRUE(Url::parse("pulsar://example.com:8080", url));
    ASSERT_EQ("example.com", url.host());
    ASSERT_EQ("pulsar", url.protocol());
    ASSERT_EQ(8080, url.port());

    ASSERT_TRUE(Url::parse("pulsar://example.com", url));
    ASSERT_EQ("example.com", url.host());
    ASSERT_EQ("pulsar", url.protocol());
    ASSERT_EQ(6650, url.port());
}
<|endoftext|>


<commit_before>#include <sstream>
#include "controller.hpp"

Controller::Controller(): n("~"), new_task(false), ac_speak("/speak", true), ac_gaze("/gaze_at_pose", true), memory_ppl(), name_dict(), person_id(-1), client_facts(n.serviceClient<facts::TellFacts>("tell_facts")), initialized(false)
{
	name_tag_sub = n.subscribe<circle_detection::detection_results_array>("/circle_detection/results_array", 1000, &Controller::tagSubscriber, this);

	rnd_walk_start = n.serviceClient<std_srvs::Empty>("/start_random_walk");
	rnd_walk_stop = n.serviceClient<std_srvs::Empty>("/stop_random_walk");

	fillDictionary();

	ros::spinOnce();
}

void Controller::startDialog()
{
	std::cerr<<"I feel like I should talk more..."<<std::endl;
	ac_speak.waitForServer();

	std::stringstream ss;
	std::string name = "";

	// get the name for that person
	std::map<int,std::string>::iterator it;
	it = name_dict.find(person_id);
	if(it != name_dict.end()) {
		name = it->second;
	} else {
		name = "unknown person. I do not recognize you"; // start an alarm?
	}

	ss << "Hi " << name << ".";

	// check how often we have seen that person before
	std::map<int,int>::iterator it_count;
	it_count = memory_ppl.find(person_id);
	if(it_count != memory_ppl.end()) {
		ss << "We have already met " << it_count->second << " times before.";

    // query the facts service
    facts::TellFacts srv;
    srv.request.person = person_id;
    if(client_facts.call(srv)) {
      // we got a new fact
      std::cerr<<"Received a new fact: " << srv.response.fact << std::endl;
      ss << srv.response.fact << std::endl;
    } else {
      std::cerr<<"Did not receive a new fact."<<std::endl;
    }
	} else {
		ss << " Welcome to the E C M R.";
		ss << " Be aware of the other robots. They are plotting an evil plan";
		ss << " against you. Especially the green one.";
		ss << " I am looking forward to seeing you again.";
		ss << std::endl;
	}

	mary_tts::maryttsGoal goal;
  	goal.text = ss.str();
  	ac_speak.sendGoal(goal);

	bool finished_before_timeout = ac_speak.waitForResult(ros::Duration(30.0));
	if (finished_before_timeout)
	{
	  actionlib::SimpleClientGoalState state = ac_speak.getState();
	  ROS_INFO("Action finished: %s",state.toString().c_str());
	}
}

void Controller::startGaze()
{
	ac_gaze.waitForServer();

	strands_gazing::GazeAtPoseGoal goal;
  	goal.runtime_sec = 0;
	goal.topic_name = "/upper_body_detector/closest_bounding_box_centre";
  	ac_gaze.sendGoal(goal);

	bool finished_before_timeout = ac_gaze.waitForResult(ros::Duration(10.0));
	if (finished_before_timeout)
	{
	  actionlib::SimpleClientGoalState state = ac_gaze.getState();
	  ROS_INFO("Action finished: %s",state.toString().c_str());
	}
}

void Controller::tagSubscriber(const circle_detection::detection_results_array::ConstPtr& _msg)
{
	if( person_id != _msg->personId )
	{
		new_task = true;
		person_id = _msg->personId;
		std::cerr<<"This is another person with the id: "<<person_id<<std::endl;
	}
}

void Controller::update()
{
	std_srvs::Empty srv;

  // initialization: start with random walking
  if(!initialized) {
    rnd_walk_start.call(srv);
    initialized=true; // will never bet to false again
  }

	if (new_task)
	{
		new_task = false;
		// update our people memory
		updatePersonSeen(person_id);

		std::cerr<<"Let's stop here for a while..."<<std::endl;
		rnd_walk_stop.call(srv);

    std::cerr<<"Look at this!"<<std::endl;
		startGaze();
		std::cerr<<"I feel active..."<<std::endl;
		startDialog();

		ac_gaze.cancelAllGoals();
		std::cerr<<"I would like to start roaming again..."<<std::endl;
		rnd_walk_start.call(srv);
	}
}

void Controller::updatePersonSeen(const int & _person_id) {
	// check whether that person exists in the memory
	std::map<int,int>::iterator it;
	it = memory_ppl.find(_person_id);
	if(it != memory_ppl.end()) {
		// increase it for a known user
		std::cerr<<"I have seen the person already "<<it->second<<" times."<<std::endl;
		memory_ppl[_person_id] = it->second++;
	} else {
		// init new user
		std::cerr<<"This is a new person to me."<<std::endl;
		memory_ppl[_person_id] = 1;
	}
}

void Controller::fillDictionary() {
	// fill the name dictionary with peoples' names
	name_dict[0] = "Bob";
	name_dict[1] = "Betty";
	name_dict[2] = "Linda";
	name_dict[3] = "Lucie";
}

int main(int argc, char** argv)
{
	ros::init(argc, argv, "controller_bev");
	Controller theController;

	ros::Rate loop_rate(5); // [Hz]

  while(ros::ok())
  {
		theController.update();

		ros::spinOnce();
    loop_rate.sleep();
  }

  return 0;
}
<commit_msg>change name of tell facts<commit_after>#include <sstream>
#include "controller.hpp"

Controller::Controller(): n("~"), new_task(false), ac_speak("/speak", true), ac_gaze("/gaze_at_pose", true), memory_ppl(), name_dict(), person_id(-1), client_facts(n.serviceClient<facts::TellFacts>("/tell_facts")), initialized(false)
{
	name_tag_sub = n.subscribe<circle_detection::detection_results_array>("/circle_detection/results_array", 1000, &Controller::tagSubscriber, this);

	rnd_walk_start = n.serviceClient<std_srvs::Empty>("/start_random_walk");
	rnd_walk_stop = n.serviceClient<std_srvs::Empty>("/stop_random_walk");

	fillDictionary();

	ros::spinOnce();
}

void Controller::startDialog()
{
	std::cerr<<"I feel like I should talk more..."<<std::endl;
	ac_speak.waitForServer();

	std::stringstream ss;
	std::string name = "";

	// get the name for that person
	std::map<int,std::string>::iterator it;
	it = name_dict.find(person_id);
	if(it != name_dict.end()) {
		name = it->second;
	} else {
		name = "unknown person. I do not recognize you"; // start an alarm?
	}

	ss << "Hi " << name << ".";

	// check how often we have seen that person before
	std::map<int,int>::iterator it_count;
	it_count = memory_ppl.find(person_id);
	if(it_count != memory_ppl.end()) {
		ss << "We have already met " << it_count->second << " times before.";

    // query the facts service
    facts::TellFacts srv;
    srv.request.person = person_id;
    if(client_facts.call(srv)) {
      // we got a new fact
      std::cerr<<"Received a new fact: " << srv.response.fact << std::endl;
      ss << srv.response.fact << std::endl;
    } else {
      std::cerr<<"Did not receive a new fact."<<std::endl;
    }
	} else {
		ss << " Welcome to the E C M R.";
		ss << " Be aware of the other robots. They are plotting an evil plan";
		ss << " against you. Especially the green one.";
		ss << " I am looking forward to seeing you again.";
		ss << std::endl;
	}

	mary_tts::maryttsGoal goal;
  	goal.text = ss.str();
  	ac_speak.sendGoal(goal);

	bool finished_before_timeout = ac_speak.waitForResult(ros::Duration(30.0));
	if (finished_before_timeout)
	{
	  actionlib::SimpleClientGoalState state = ac_speak.getState();
	  ROS_INFO("Action finished: %s",state.toString().c_str());
	}
}

void Controller::startGaze()
{
	ac_gaze.waitForServer();

	strands_gazing::GazeAtPoseGoal goal;
  	goal.runtime_sec = 0;
	goal.topic_name = "/upper_body_detector/closest_bounding_box_centre";
  	ac_gaze.sendGoal(goal);

	bool finished_before_timeout = ac_gaze.waitForResult(ros::Duration(10.0));
	if (finished_before_timeout)
	{
	  actionlib::SimpleClientGoalState state = ac_gaze.getState();
	  ROS_INFO("Action finished: %s",state.toString().c_str());
	}
}

void Controller::tagSubscriber(const circle_detection::detection_results_array::ConstPtr& _msg)
{
	if( person_id != _msg->personId )
	{
		new_task = true;
		person_id = _msg->personId;
		std::cerr<<"This is another person with the id: "<<person_id<<std::endl;
	}
}

void Controller::update()
{
	std_srvs::Empty srv;

  // initialization: start with random walking
  if(!initialized) {
    rnd_walk_start.call(srv);
    initialized=true; // will never bet to false again
  }

	if (new_task)
	{
		new_task = false;
		// update our people memory
		updatePersonSeen(person_id);

		std::cerr<<"Let's stop here for a while..."<<std::endl;
		rnd_walk_stop.call(srv);

    std::cerr<<"Look at this!"<<std::endl;
		startGaze();
		std::cerr<<"I feel active..."<<std::endl;
		startDialog();

		ac_gaze.cancelAllGoals();
		std::cerr<<"I would like to start roaming again..."<<std::endl;
		rnd_walk_start.call(srv);
	}
}

void Controller::updatePersonSeen(const int & _person_id) {
	// check whether that person exists in the memory
	std::map<int,int>::iterator it;
	it = memory_ppl.find(_person_id);
	if(it != memory_ppl.end()) {
		// increase it for a known user
		std::cerr<<"I have seen the person already "<<it->second<<" times."<<std::endl;
		memory_ppl[_person_id] = it->second++;
	} else {
		// init new user
		std::cerr<<"This is a new person to me."<<std::endl;
		memory_ppl[_person_id] = 1;
	}
}

void Controller::fillDictionary() {
	// fill the name dictionary with peoples' names
	name_dict[0] = "Bob";
	name_dict[1] = "Betty";
	name_dict[2] = "Linda";
	name_dict[3] = "Lucie";
}

int main(int argc, char** argv)
{
	ros::init(argc, argv, "controller_bev");
	Controller theController;

	ros::Rate loop_rate(5); // [Hz]

  while(ros::ok())
  {
		theController.update();

		ros::spinOnce();
    loop_rate.sleep();
  }

  return 0;
}
<|endoftext|>


<commit_before>#include "detectors.h"

#include <iostream>
#include <numeric>
#include <algorithm>
#include <cmath>
#include <limits>

using namespace std;

#include "poptemplate.h"

static const bool kDelayMatch = false;

static const int kBlockSize = DETECTORS_BLOCK_SIZE;
static const int kLogBlockSize = 9;
static const int kSpectrumSize = kBlockSize/2;
static const int kWindowSize = kBlockSize;

static const int kNumSteps = 4;
static const int kStepSize = kBlockSize / kNumSteps;

static const size_t kMainBandLow = 40;
static const size_t kMainBandHi = 100;
static const size_t kOptionalBandHi = 180;

static const size_t kLowerBandLow = 3;
static const size_t kLowerBandHi = kMainBandLow;
static const size_t kUpperBandLo = kOptionalBandHi;
static const size_t kUpperBandHi = kSpectrumSize;

static const float kDefaultLowPassWeight = 0.6;
static const int kSpeechShadowTime = 100;
static const float kSpeechThresh = 0.5;

Detectors::Detectors() {
  m_overlapBuffer = new float[kBlockSize * 2];

  // === Tss Detection
  m_sensitivity = 5.0;
  m_hysterisisFactor = 0.4;
  m_minFrames = 20;
  m_minFramesLong = 100;
  m_lowPassWeight = kDefaultLowPassWeight;

  // === Pop detection
  m_startBin = 2;
  m_maxShiftDown = 4;
  m_maxShiftUp = 2;
  m_popSensitivity = 8.5;
  m_framesSincePop = 0;

  // debugLog = new std::ofstream("/Users/tristan/misc/popclick.log");

  // === FFT
  m_inReal = new float[kBlockSize];
  m_splitData.realp = new float[kSpectrumSize];
  m_splitData.imagp = new float[kSpectrumSize];
  m_window = new float[kWindowSize];
  memset(m_window, 0, sizeof(float) * kWindowSize);
  vDSP_hann_window(m_window, kWindowSize, vDSP_HANN_NORM);

  m_fftSetup = vDSP_create_fftsetup(kLogBlockSize, FFT_RADIX2);
}

Detectors::~Detectors() {
  delete[] m_overlapBuffer;

  delete[] m_inReal;
  delete[] m_splitData.realp;
  delete[] m_splitData.imagp;
  delete[] m_window;
  // delete debugLog;

  vDSP_destroy_fftsetup(m_fftSetup);
}

bool Detectors::initialise() {
  // Real initialisation work goes here!
  m_savedOtherBands = 0.0002;
  m_consecutiveMatches = 0;
  m_framesSinceSpeech = 1000;
  m_framesSinceMatch = 1000;
  m_lowPassBuffer.resize(kSpectrumSize, 0.0);

  m_spectrum.resize(kSpectrumSize, 0.0);
  m_popBuffer.clear();
  for(unsigned i = 0; i < kBufferSize; ++i) {
    m_popBuffer.push_back(0.0);
  }

  return true;
}

int Detectors::process(const float *buffer) {
  // return processChunk(buffer);
  // copy last frame to start of the buffer
  std::copy(m_overlapBuffer+kBlockSize, m_overlapBuffer+(kBlockSize*2), m_overlapBuffer);
  // copy new input to the second half of the overlap buffer
  std::copy(buffer,buffer+kBlockSize,m_overlapBuffer+kBlockSize);

  int result = 0;
  for(int i = 0; i < kNumSteps; ++i) {
    float *ptr = m_overlapBuffer+((i+1)*kStepSize);
    result |= processChunk(ptr);
  }
  return result;
}

void Detectors::doFFT(const float *buffer) {
  vDSP_vmul(buffer, 1, m_window, 1, m_inReal, 1, kBlockSize);
  vDSP_ctoz(reinterpret_cast<DSPComplex*>(m_inReal), 2, &m_splitData, 1, kSpectrumSize);
  vDSP_fft_zrip(m_fftSetup, &m_splitData, 1, kLogBlockSize, FFT_FORWARD);
  m_splitData.imagp[0] = 0.0f;

  float scale = 1.0f / static_cast<float>(2 * kBlockSize);
  vDSP_vsmul(m_splitData.realp, 1, &scale, m_splitData.realp, 1, kSpectrumSize);
  vDSP_vsmul(m_splitData.imagp, 1, &scale, m_splitData.imagp, 1, kSpectrumSize);
}

int Detectors::processChunk(const float *buffer) {
  doFFT(buffer);

  int result = 0;
  size_t n = kSpectrumSize;

  for (size_t i = 0; i < n; ++i) {
    float real = m_splitData.realp[i];
    float imag = m_splitData.imagp[i];
    float newVal = real * real + imag * imag;
    m_spectrum[i] = newVal;
    m_lowPassBuffer[i] = m_lowPassBuffer[i]*(1.0f-m_lowPassWeight) + newVal*m_lowPassWeight;

    // infinite values happen non-deterministically, probably due to glitchy audio input at start of recording
    // but inifinities it could mess up things forever
    if(m_lowPassBuffer[i] >= numeric_limits<float>::infinity()) {
      std::fill(m_lowPassBuffer.begin(), m_lowPassBuffer.end(), 0.0f);
      return 0; // discard the frame, it's probably garbage
    }
  }

  float lowerBand = avgBand(m_lowPassBuffer, kLowerBandLow, kLowerBandHi);
  float mainBand = avgBand(m_lowPassBuffer, kMainBandLow, kMainBandHi);
  float upperBand = avgBand(m_lowPassBuffer, kUpperBandLo, kUpperBandHi);

  m_framesSinceSpeech += 1;
  if(lowerBand > kSpeechThresh) {
    m_framesSinceSpeech = 0;
  }

  float debugMarker = 0.0002;
  float matchiness = mainBand / ((lowerBand+upperBand)/2.0f);
  bool outOfShadow = m_framesSinceSpeech > kSpeechShadowTime;
  int immediateMatchFrame = kDelayMatch ? m_minFramesLong : m_minFrames;
  m_framesSinceMatch += 1;
  if(((matchiness >= m_sensitivity) ||
      (m_consecutiveMatches > 0 && matchiness >= m_sensitivity*m_hysterisisFactor) ||
      (m_consecutiveMatches > immediateMatchFrame && (mainBand/m_savedOtherBands) >= m_sensitivity*m_hysterisisFactor*0.5f))
     && outOfShadow) {
    debugMarker = 0.01;
    // second one in double "tss" came earlier than trigger timer
    if(kDelayMatch && m_consecutiveMatches == 0 && m_framesSinceMatch <= m_minFramesLong) {
      result |= TSS_START_CODE;
      result |= TSS_STOP_CODE;
      m_framesSinceMatch = 1000;
    }

    m_consecutiveMatches += 1;
    if(kDelayMatch && m_consecutiveMatches == m_minFrames) {
      m_framesSinceMatch = m_consecutiveMatches;
    } else if(m_consecutiveMatches == immediateMatchFrame) {
      debugMarker = 1.0;
      result |= TSS_START_CODE;
      m_savedOtherBands = ((lowerBand+upperBand)/2.0f);
    }
  } else {
    bool delayedMatch = kDelayMatch && (m_framesSinceMatch == m_minFramesLong && outOfShadow);
    if(delayedMatch) {
      result |= TSS_START_CODE;
    }
    if(m_consecutiveMatches >= immediateMatchFrame || delayedMatch) {
      debugMarker = 2.0;
      result |= TSS_STOP_CODE;
    }
    m_consecutiveMatches = 0;
  }

  // ===================== Pop Detection =================================
  // update buffer forward one time step
  for(unsigned i = 0; i < kBufferPrimaryHeight; ++i) {
    m_popBuffer.pop_front();
    m_popBuffer.push_back(m_spectrum[i]);
  }
  // high frequencies aren't useful so we bin them all together
  m_popBuffer.pop_front();
  float highSum = accumulate(m_spectrum.begin()+kBufferPrimaryHeight,m_spectrum.end(),0.0);
  m_popBuffer.push_back(highSum);

  std::deque<float>::iterator maxIt = max_element(m_popBuffer.begin(), m_popBuffer.end());
  float minDiff = 10000000.0;
  for(int i = -m_maxShiftUp; i < m_maxShiftDown; ++i) {
    float diff = templateDiff(*maxIt, i);
    if(diff < minDiff) minDiff = diff;
  }

  m_framesSincePop += 1;
  if(minDiff < m_popSensitivity && m_framesSincePop > 15) {
    result |= POP_CODE; // Detected pop
    m_framesSincePop = 0;
  }

  // *debugLog << lowerBand << ' ' << mainBand << ' ' << optionalBand << ' ' << upperBand << '-' << matchiness << ' ' << debugMarker << std::endl;
  return result;
}

float Detectors::avgBand(std::vector<float> &frame, size_t low, size_t hi) {
  float sum = 0;
  for (size_t i = low; i < hi; ++i) {
    sum += frame[i];
  }
  return sum / (hi - low);
}

float Detectors::templateAt(int i, int shift) {
  int bin = i % kBufferHeight;
  if(i % kBufferHeight >= kBufferPrimaryHeight) {
    return kPopTemplate[i]/kPopTemplateMax;
  }
  if(bin+shift < 0 || bin+shift >= kBufferPrimaryHeight) {
    return 0.0;
  }
  return kPopTemplate[i+shift]/kPopTemplateMax;
}

float Detectors::diffCol(int templStart, int bufStart, float maxVal, int shift) {
  float diff = 0;
  for(unsigned i = m_startBin; i < kBufferHeight; ++i) {
    float d = templateAt(templStart+i, shift) - m_popBuffer[bufStart+i]/maxVal;
    diff += abs(d);
  }
  return diff;
}

float Detectors::templateDiff(float maxVal, int shift) {
  float diff = 0;
  for(unsigned i = 0; i < kBufferSize; i += kBufferHeight) {
    diff += diffCol(i,i, maxVal,shift);
  }
  return diff;
}

extern "C" {
  detectors_t *detectors_new() {
    Detectors *dets = new Detectors();
    dets->initialise();
    return reinterpret_cast<detectors_t*>(dets);
  }
  void detectors_free(detectors_t *detectors) {
    Detectors *dets = reinterpret_cast<Detectors*>(detectors);
    delete dets;
  }
  int detectors_process(detectors_t *detectors, const float *buffer) {
    Detectors *dets = reinterpret_cast<Detectors*>(detectors);
    return dets->process(buffer);
  }
}

<commit_msg>Fix #include in hs.noises for the correct filename case<commit_after>#include "detectors.h"

#include <iostream>
#include <numeric>
#include <algorithm>
#include <cmath>
#include <limits>

using namespace std;

#include "popTemplate.h"

static const bool kDelayMatch = false;

static const int kBlockSize = DETECTORS_BLOCK_SIZE;
static const int kLogBlockSize = 9;
static const int kSpectrumSize = kBlockSize/2;
static const int kWindowSize = kBlockSize;

static const int kNumSteps = 4;
static const int kStepSize = kBlockSize / kNumSteps;

static const size_t kMainBandLow = 40;
static const size_t kMainBandHi = 100;
static const size_t kOptionalBandHi = 180;

static const size_t kLowerBandLow = 3;
static const size_t kLowerBandHi = kMainBandLow;
static const size_t kUpperBandLo = kOptionalBandHi;
static const size_t kUpperBandHi = kSpectrumSize;

static const float kDefaultLowPassWeight = 0.6;
static const int kSpeechShadowTime = 100;
static const float kSpeechThresh = 0.5;

Detectors::Detectors() {
  m_overlapBuffer = new float[kBlockSize * 2];

  // === Tss Detection
  m_sensitivity = 5.0;
  m_hysterisisFactor = 0.4;
  m_minFrames = 20;
  m_minFramesLong = 100;
  m_lowPassWeight = kDefaultLowPassWeight;

  // === Pop detection
  m_startBin = 2;
  m_maxShiftDown = 4;
  m_maxShiftUp = 2;
  m_popSensitivity = 8.5;
  m_framesSincePop = 0;

  // debugLog = new std::ofstream("/Users/tristan/misc/popclick.log");

  // === FFT
  m_inReal = new float[kBlockSize];
  m_splitData.realp = new float[kSpectrumSize];
  m_splitData.imagp = new float[kSpectrumSize];
  m_window = new float[kWindowSize];
  memset(m_window, 0, sizeof(float) * kWindowSize);
  vDSP_hann_window(m_window, kWindowSize, vDSP_HANN_NORM);

  m_fftSetup = vDSP_create_fftsetup(kLogBlockSize, FFT_RADIX2);
}

Detectors::~Detectors() {
  delete[] m_overlapBuffer;

  delete[] m_inReal;
  delete[] m_splitData.realp;
  delete[] m_splitData.imagp;
  delete[] m_window;
  // delete debugLog;

  vDSP_destroy_fftsetup(m_fftSetup);
}

bool Detectors::initialise() {
  // Real initialisation work goes here!
  m_savedOtherBands = 0.0002;
  m_consecutiveMatches = 0;
  m_framesSinceSpeech = 1000;
  m_framesSinceMatch = 1000;
  m_lowPassBuffer.resize(kSpectrumSize, 0.0);

  m_spectrum.resize(kSpectrumSize, 0.0);
  m_popBuffer.clear();
  for(unsigned i = 0; i < kBufferSize; ++i) {
    m_popBuffer.push_back(0.0);
  }

  return true;
}

int Detectors::process(const float *buffer) {
  // return processChunk(buffer);
  // copy last frame to start of the buffer
  std::copy(m_overlapBuffer+kBlockSize, m_overlapBuffer+(kBlockSize*2), m_overlapBuffer);
  // copy new input to the second half of the overlap buffer
  std::copy(buffer,buffer+kBlockSize,m_overlapBuffer+kBlockSize);

  int result = 0;
  for(int i = 0; i < kNumSteps; ++i) {
    float *ptr = m_overlapBuffer+((i+1)*kStepSize);
    result |= processChunk(ptr);
  }
  return result;
}

void Detectors::doFFT(const float *buffer) {
  vDSP_vmul(buffer, 1, m_window, 1, m_inReal, 1, kBlockSize);
  vDSP_ctoz(reinterpret_cast<DSPComplex*>(m_inReal), 2, &m_splitData, 1, kSpectrumSize);
  vDSP_fft_zrip(m_fftSetup, &m_splitData, 1, kLogBlockSize, FFT_FORWARD);
  m_splitData.imagp[0] = 0.0f;

  float scale = 1.0f / static_cast<float>(2 * kBlockSize);
  vDSP_vsmul(m_splitData.realp, 1, &scale, m_splitData.realp, 1, kSpectrumSize);
  vDSP_vsmul(m_splitData.imagp, 1, &scale, m_splitData.imagp, 1, kSpectrumSize);
}

int Detectors::processChunk(const float *buffer) {
  doFFT(buffer);

  int result = 0;
  size_t n = kSpectrumSize;

  for (size_t i = 0; i < n; ++i) {
    float real = m_splitData.realp[i];
    float imag = m_splitData.imagp[i];
    float newVal = real * real + imag * imag;
    m_spectrum[i] = newVal;
    m_lowPassBuffer[i] = m_lowPassBuffer[i]*(1.0f-m_lowPassWeight) + newVal*m_lowPassWeight;

    // infinite values happen non-deterministically, probably due to glitchy audio input at start of recording
    // but inifinities it could mess up things forever
    if(m_lowPassBuffer[i] >= numeric_limits<float>::infinity()) {
      std::fill(m_lowPassBuffer.begin(), m_lowPassBuffer.end(), 0.0f);
      return 0; // discard the frame, it's probably garbage
    }
  }

  float lowerBand = avgBand(m_lowPassBuffer, kLowerBandLow, kLowerBandHi);
  float mainBand = avgBand(m_lowPassBuffer, kMainBandLow, kMainBandHi);
  float upperBand = avgBand(m_lowPassBuffer, kUpperBandLo, kUpperBandHi);

  m_framesSinceSpeech += 1;
  if(lowerBand > kSpeechThresh) {
    m_framesSinceSpeech = 0;
  }

  float debugMarker = 0.0002;
  float matchiness = mainBand / ((lowerBand+upperBand)/2.0f);
  bool outOfShadow = m_framesSinceSpeech > kSpeechShadowTime;
  int immediateMatchFrame = kDelayMatch ? m_minFramesLong : m_minFrames;
  m_framesSinceMatch += 1;
  if(((matchiness >= m_sensitivity) ||
      (m_consecutiveMatches > 0 && matchiness >= m_sensitivity*m_hysterisisFactor) ||
      (m_consecutiveMatches > immediateMatchFrame && (mainBand/m_savedOtherBands) >= m_sensitivity*m_hysterisisFactor*0.5f))
     && outOfShadow) {
    debugMarker = 0.01;
    // second one in double "tss" came earlier than trigger timer
    if(kDelayMatch && m_consecutiveMatches == 0 && m_framesSinceMatch <= m_minFramesLong) {
      result |= TSS_START_CODE;
      result |= TSS_STOP_CODE;
      m_framesSinceMatch = 1000;
    }

    m_consecutiveMatches += 1;
    if(kDelayMatch && m_consecutiveMatches == m_minFrames) {
      m_framesSinceMatch = m_consecutiveMatches;
    } else if(m_consecutiveMatches == immediateMatchFrame) {
      debugMarker = 1.0;
      result |= TSS_START_CODE;
      m_savedOtherBands = ((lowerBand+upperBand)/2.0f);
    }
  } else {
    bool delayedMatch = kDelayMatch && (m_framesSinceMatch == m_minFramesLong && outOfShadow);
    if(delayedMatch) {
      result |= TSS_START_CODE;
    }
    if(m_consecutiveMatches >= immediateMatchFrame || delayedMatch) {
      debugMarker = 2.0;
      result |= TSS_STOP_CODE;
    }
    m_consecutiveMatches = 0;
  }

  // ===================== Pop Detection =================================
  // update buffer forward one time step
  for(unsigned i = 0; i < kBufferPrimaryHeight; ++i) {
    m_popBuffer.pop_front();
    m_popBuffer.push_back(m_spectrum[i]);
  }
  // high frequencies aren't useful so we bin them all together
  m_popBuffer.pop_front();
  float highSum = accumulate(m_spectrum.begin()+kBufferPrimaryHeight,m_spectrum.end(),0.0);
  m_popBuffer.push_back(highSum);

  std::deque<float>::iterator maxIt = max_element(m_popBuffer.begin(), m_popBuffer.end());
  float minDiff = 10000000.0;
  for(int i = -m_maxShiftUp; i < m_maxShiftDown; ++i) {
    float diff = templateDiff(*maxIt, i);
    if(diff < minDiff) minDiff = diff;
  }

  m_framesSincePop += 1;
  if(minDiff < m_popSensitivity && m_framesSincePop > 15) {
    result |= POP_CODE; // Detected pop
    m_framesSincePop = 0;
  }

  // *debugLog << lowerBand << ' ' << mainBand << ' ' << optionalBand << ' ' << upperBand << '-' << matchiness << ' ' << debugMarker << std::endl;
  return result;
}

float Detectors::avgBand(std::vector<float> &frame, size_t low, size_t hi) {
  float sum = 0;
  for (size_t i = low; i < hi; ++i) {
    sum += frame[i];
  }
  return sum / (hi - low);
}

float Detectors::templateAt(int i, int shift) {
  int bin = i % kBufferHeight;
  if(i % kBufferHeight >= kBufferPrimaryHeight) {
    return kPopTemplate[i]/kPopTemplateMax;
  }
  if(bin+shift < 0 || bin+shift >= kBufferPrimaryHeight) {
    return 0.0;
  }
  return kPopTemplate[i+shift]/kPopTemplateMax;
}

float Detectors::diffCol(int templStart, int bufStart, float maxVal, int shift) {
  float diff = 0;
  for(unsigned i = m_startBin; i < kBufferHeight; ++i) {
    float d = templateAt(templStart+i, shift) - m_popBuffer[bufStart+i]/maxVal;
    diff += abs(d);
  }
  return diff;
}

float Detectors::templateDiff(float maxVal, int shift) {
  float diff = 0;
  for(unsigned i = 0; i < kBufferSize; i += kBufferHeight) {
    diff += diffCol(i,i, maxVal,shift);
  }
  return diff;
}

extern "C" {
  detectors_t *detectors_new() {
    Detectors *dets = new Detectors();
    dets->initialise();
    return reinterpret_cast<detectors_t*>(dets);
  }
  void detectors_free(detectors_t *detectors) {
    Detectors *dets = reinterpret_cast<Detectors*>(detectors);
    delete dets;
  }
  int detectors_process(detectors_t *detectors, const float *buffer) {
    Detectors *dets = reinterpret_cast<Detectors*>(detectors);
    return dets->process(buffer);
  }
}

<|endoftext|>


<commit_before>/*
 * Copyright (c) 2003-2006 The Regents of The University of Michigan
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * Authors: Nathan Binkert
 */

#ifndef __SIM_PSEUDO_INST_HH__
#define __SIM_PSEUDO_INST_HH__

class ThreadContext;

//We need the "Tick" and "Addr" data types from here
#include "base/types.hh"

namespace PseudoInst {

/**
 * @todo these externs are only here for a hack in fullCPU::takeOver...
 */
extern bool doStatisticsInsts;
extern bool doCheckpointInsts;
extern bool doQuiesce;

void arm(ThreadContext *tc);
void quiesce(ThreadContext *tc);
void quiesceSkip(ThreadContext *tc);
void quiesceNs(ThreadContext *tc, uint64_t ns);
void quiesceCycles(ThreadContext *tc, uint64_t cycles);
uint64_t quiesceTime(ThreadContext *tc);
uint64_t readfile(ThreadContext *tc, Addr vaddr, uint64_t len,
    uint64_t offset);
uint64_t writefile(ThreadContext *tc, Addr vaddr, uint64_t len,
    uint64_t offset, Addr filenameAddr);
void loadsymbol(ThreadContext *xc);
void addsymbol(ThreadContext *tc, Addr addr, Addr symbolAddr);
uint64_t initParam(ThreadContext *xc);
uint64_t rpns(ThreadContext *tc);
void wakeCPU(ThreadContext *tc, uint64_t cpuid);
void m5exit(ThreadContext *tc, Tick delay);
void resetstats(ThreadContext *tc, Tick delay, Tick period);
void dumpstats(ThreadContext *tc, Tick delay, Tick period);
void dumpresetstats(ThreadContext *tc, Tick delay, Tick period);
void m5checkpoint(ThreadContext *tc, Tick delay, Tick period);
void debugbreak(ThreadContext *tc);
void switchcpu(ThreadContext *tc);
void workbegin(ThreadContext *tc, uint64_t workid, uint64_t threadid);
void workend(ThreadContext *tc, uint64_t workid, uint64_t threadid);

} // namespace PseudoInst

#endif // __SIM_PSEUDO_INST_HH__
<commit_msg>sim: Remove unused variables<commit_after>/*
 * Copyright (c) 2003-2006 The Regents of The University of Michigan
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * Authors: Nathan Binkert
 */

#ifndef __SIM_PSEUDO_INST_HH__
#define __SIM_PSEUDO_INST_HH__

class ThreadContext;

//We need the "Tick" and "Addr" data types from here
#include "base/types.hh"

namespace PseudoInst {

void arm(ThreadContext *tc);
void quiesce(ThreadContext *tc);
void quiesceSkip(ThreadContext *tc);
void quiesceNs(ThreadContext *tc, uint64_t ns);
void quiesceCycles(ThreadContext *tc, uint64_t cycles);
uint64_t quiesceTime(ThreadContext *tc);
uint64_t readfile(ThreadContext *tc, Addr vaddr, uint64_t len,
    uint64_t offset);
uint64_t writefile(ThreadContext *tc, Addr vaddr, uint64_t len,
    uint64_t offset, Addr filenameAddr);
void loadsymbol(ThreadContext *xc);
void addsymbol(ThreadContext *tc, Addr addr, Addr symbolAddr);
uint64_t initParam(ThreadContext *xc);
uint64_t rpns(ThreadContext *tc);
void wakeCPU(ThreadContext *tc, uint64_t cpuid);
void m5exit(ThreadContext *tc, Tick delay);
void resetstats(ThreadContext *tc, Tick delay, Tick period);
void dumpstats(ThreadContext *tc, Tick delay, Tick period);
void dumpresetstats(ThreadContext *tc, Tick delay, Tick period);
void m5checkpoint(ThreadContext *tc, Tick delay, Tick period);
void debugbreak(ThreadContext *tc);
void switchcpu(ThreadContext *tc);
void workbegin(ThreadContext *tc, uint64_t workid, uint64_t threadid);
void workend(ThreadContext *tc, uint64_t workid, uint64_t threadid);

} // namespace PseudoInst

#endif // __SIM_PSEUDO_INST_HH__
<|endoftext|>