<commit_before>/*
 * Software License Agreement (BSD License)
 *
 *  Copyright (c) 2010, Willow Garage, Inc.
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions
 *  are met:
 *
 *   * Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 *   * Redistributions in binary form must reproduce the above
 *     copyright notice, this list of conditions and the following
 *     disclaimer in the documentation and/or other materials provided
 *     with the distribution.
 *   * Neither the name of Willow Garage, Inc. nor the names of its
 *     contributors may be used to endorse or promote products derived
 *     from this software without specific prior written permission.
 *
 *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 *  FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 *  COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 *  BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 *  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 *  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 *  LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 *  ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 *  POSSIBILITY OF SUCH DAMAGE.
 *
 * $Id$
 *
 */

#ifndef PCL_FILTERS_IMPL_STATISTICAL_OUTLIER_REMOVAL_H_
#define PCL_FILTERS_IMPL_STATISTICAL_OUTLIER_REMOVAL_H_

#include "pcl/filters/statistical_outlier_removal.h"

//////////////////////////////////////////////////////////////////////////////////////////////
template <typename PointT> void
pcl::StatisticalOutlierRemoval<PointT>::applyFilter (PointCloud &output)
{
  if (std_mul_ == 0.0)
  {
    PCL_ERROR ("[pcl::%s::applyFilter] Standard deviation multiplier not set!\n", getClassName ().c_str ());
    output.width = output.height = 0;
    output.points.clear ();
    return;
  }

  // Initialize the spatial locator
  //initTree (spatial_locator_type_, tree_, k_);

  // TODO: fix this
  tree_.reset (new KdTreeFLANN<PointT> ());

  // Send the input dataset to the spatial locator
  tree_->setInputCloud (input_);

  // Allocate enough space to hold the results
  std::vector<int> nn_indices (mean_k_);
  std::vector<float> nn_dists (mean_k_);

  std::vector<float> distances (indices_->size ());
  // Go over all the points and calculate the mean or smallest distance
  for (size_t cp = 0; cp < indices_->size (); ++cp)
  {
    if (!pcl_isfinite (input_->points[(*indices_)[cp]].x) ||
        !pcl_isfinite (input_->points[(*indices_)[cp]].y) ||
        !pcl_isfinite (input_->points[(*indices_)[cp]].z))
    {
      distances[cp] = 0;
      continue;
    }

    if (tree_->nearestKSearch ((*indices_)[cp], mean_k_, nn_indices, nn_dists) == 0)
    {
      distances[cp] = 0;
      PCL_WARN ("[pcl::%s::applyFilter] Searching for the closest %d neighbors failed.\n", getClassName ().c_str (), mean_k_);
      continue;
    }

    // Minimum distance (if mean_k_ == 2) or mean distance
    double dist_sum = 0;
    for (int j = 1; j < mean_k_; ++j)
      dist_sum += sqrt (nn_dists[j]);
    distances[cp] = dist_sum / (mean_k_-1);
  }

  // Estimate the mean and the standard deviation of the distance vector
  double mean, stddev;
  getMeanStd (distances, mean, stddev);
  double distance_threshold = mean + std_mul_ * stddev; // a distance that is bigger than this signals an outlier

  output.points.resize (input_->points.size ());      // reserve enough space
  removed_indices_->resize (input_->points.size ());

  // Build a new cloud by neglecting outliers
  int nr_p = 0;
  int nr_removed_p = 0;

  for (size_t cp = 0; cp < indices_->size (); ++cp)
  {
    if (negative_)
    {
      if (distances[cp] <= distance_threshold)
      {
        if (extract_removed_indices_)
        {
          (*removed_indices_)[nr_removed_p] = cp;
          nr_removed_p++;
        }
        continue;
      }
    }
    else
    {
      if (distances[cp] > distance_threshold)
      {
        if (extract_removed_indices_)
        {
          (*removed_indices_)[nr_removed_p] = cp;
          nr_removed_p++;
        }
        continue;
      }
    }

    output.points[nr_p++] = input_->points[(*indices_)[cp]];
  }

  output.points.resize (nr_p);
  output.width  = nr_p;
  output.height = 1;
  output.is_dense = true; // nearestKSearch filters invalid points

  removed_indices_->resize (nr_removed_p);
}

#define PCL_INSTANTIATE_StatisticalOutlierRemoval(T) template class PCL_EXPORTS pcl::StatisticalOutlierRemoval<T>;

#endif    // PCL_FILTERS_IMPL_STATISTICAL_OUTLIER_REMOVAL_H_

<commit_msg>we do not care for the results to be sorted when using StatisticalAnalysis outlier removal<commit_after>/*
 * Software License Agreement (BSD License)
 *
 *  Copyright (c) 2010, Willow Garage, Inc.
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions
 *  are met:
 *
 *   * Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 *   * Redistributions in binary form must reproduce the above
 *     copyright notice, this list of conditions and the following
 *     disclaimer in the documentation and/or other materials provided
 *     with the distribution.
 *   * Neither the name of Willow Garage, Inc. nor the names of its
 *     contributors may be used to endorse or promote products derived
 *     from this software without specific prior written permission.
 *
 *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 *  FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 *  COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 *  BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 *  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 *  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 *  LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 *  ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 *  POSSIBILITY OF SUCH DAMAGE.
 *
 * $Id$
 *
 */

#ifndef PCL_FILTERS_IMPL_STATISTICAL_OUTLIER_REMOVAL_H_
#define PCL_FILTERS_IMPL_STATISTICAL_OUTLIER_REMOVAL_H_

#include "pcl/filters/statistical_outlier_removal.h"

//////////////////////////////////////////////////////////////////////////////////////////////
template <typename PointT> void
pcl::StatisticalOutlierRemoval<PointT>::applyFilter (PointCloud &output)
{
  if (std_mul_ == 0.0)
  {
    PCL_ERROR ("[pcl::%s::applyFilter] Standard deviation multiplier not set!\n", getClassName ().c_str ());
    output.width = output.height = 0;
    output.points.clear ();
    return;
  }

  // Initialize the spatial locator
  //initTree (spatial_locator_type_, tree_, k_);

  // TODO: fix this
  tree_.reset (new KdTreeFLANN<PointT> (false));

  // Send the input dataset to the spatial locator
  tree_->setInputCloud (input_);

  // Allocate enough space to hold the results
  std::vector<int> nn_indices (mean_k_);
  std::vector<float> nn_dists (mean_k_);

  std::vector<float> distances (indices_->size ());
  // Go over all the points and calculate the mean or smallest distance
  for (size_t cp = 0; cp < indices_->size (); ++cp)
  {
    if (!pcl_isfinite (input_->points[(*indices_)[cp]].x) ||
        !pcl_isfinite (input_->points[(*indices_)[cp]].y) ||
        !pcl_isfinite (input_->points[(*indices_)[cp]].z))
    {
      distances[cp] = 0;
      continue;
    }

    if (tree_->nearestKSearch ((*indices_)[cp], mean_k_, nn_indices, nn_dists) == 0)
    {
      distances[cp] = 0;
      PCL_WARN ("[pcl::%s::applyFilter] Searching for the closest %d neighbors failed.\n", getClassName ().c_str (), mean_k_);
      continue;
    }

    // Minimum distance (if mean_k_ == 2) or mean distance
    double dist_sum = 0;
    for (int j = 1; j < mean_k_; ++j)
      dist_sum += sqrt (nn_dists[j]);
    distances[cp] = dist_sum / (mean_k_-1);
  }

  // Estimate the mean and the standard deviation of the distance vector
  double mean, stddev;
  getMeanStd (distances, mean, stddev);
  double distance_threshold = mean + std_mul_ * stddev; // a distance that is bigger than this signals an outlier

  output.points.resize (input_->points.size ());      // reserve enough space
  removed_indices_->resize (input_->points.size ());

  // Build a new cloud by neglecting outliers
  int nr_p = 0;
  int nr_removed_p = 0;

  for (size_t cp = 0; cp < indices_->size (); ++cp)
  {
    if (negative_)
    {
      if (distances[cp] <= distance_threshold)
      {
        if (extract_removed_indices_)
        {
          (*removed_indices_)[nr_removed_p] = cp;
          nr_removed_p++;
        }
        continue;
      }
    }
    else
    {
      if (distances[cp] > distance_threshold)
      {
        if (extract_removed_indices_)
        {
          (*removed_indices_)[nr_removed_p] = cp;
          nr_removed_p++;
        }
        continue;
      }
    }

    output.points[nr_p++] = input_->points[(*indices_)[cp]];
  }

  output.points.resize (nr_p);
  output.width  = nr_p;
  output.height = 1;
  output.is_dense = true; // nearestKSearch filters invalid points

  removed_indices_->resize (nr_removed_p);
}

#define PCL_INSTANTIATE_StatisticalOutlierRemoval(T) template class PCL_EXPORTS pcl::StatisticalOutlierRemoval<T>;

#endif    // PCL_FILTERS_IMPL_STATISTICAL_OUTLIER_REMOVAL_H_

<|endoftext|>


<commit_before>/* Copyright (C) 2000-2003 MySQL AB

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */

/*
  UNION  of select's
  UNION's  were introduced by Monty and Sinisa <sinisa@mysql.com>
*/

#include "mysql_priv.h"
#include "sql_select.h"

int mysql_union(THD *thd, LEX *lex, select_result *result,
		SELECT_LEX_UNIT *unit, bool tables_and_fields_initied)
{
  DBUG_ENTER("mysql_union");
  int res= 0;
  if (!(res= unit->prepare(thd, result, tables_and_fields_initied)))
    res= unit->exec();
  res|= unit->cleanup();
  DBUG_RETURN(res);
}

/***************************************************************************
** store records in temporary table for UNION
***************************************************************************/

select_union::select_union(TABLE *table_par)
  :table(table_par), not_describe(0)
{
  bzero((char*) &info,sizeof(info));
  /*
    We can always use DUP_IGNORE because the temporary table will only
    contain a unique key if we are using not using UNION ALL
  */
  info.handle_duplicates= DUP_IGNORE;
}

select_union::~select_union()
{
}

int select_union::prepare(List<Item> &list, SELECT_LEX_UNIT *u)
{
  unit= u;
  if (not_describe && list.elements != table->fields)
  {
    my_message(ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT,
	       ER(ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT),MYF(0));
    return -1;
  }
  return 0;
}

bool select_union::send_data(List<Item> &values)
{
  if (unit->offset_limit_cnt)
  {						// using limit offset,count
    unit->offset_limit_cnt--;
    return 0;
  }
  fill_record(table->field, values, 1);
  if (thd->net.report_error || write_record(table,&info))
  {
    if (thd->net.last_errno == ER_RECORD_FILE_FULL)
    {
      thd->clear_error(); // do not report user about table overflow
      if (create_myisam_from_heap(thd, table, &tmp_table_param,
				  info.last_errno, 0))
	return 1;
    }
    else
      return 1;
  }
  return 0;
}

bool select_union::send_eof()
{
  return 0;
}

bool select_union::flush()
{
  int error;
  if ((error=table->file->extra(HA_EXTRA_NO_CACHE)))
  {
    table->file->print_error(error,MYF(0));
    ::send_error(thd);
    return 1;
  }
  return 0;
}

int st_select_lex_unit::prepare(THD *thd, select_result *sel_result,
				bool tables_and_fields_initied)
{
  SELECT_LEX_NODE *lex_select_save= thd->lex.current_select;
  SELECT_LEX *select_cursor;
  DBUG_ENTER("st_select_lex_unit::prepare");

  if (prepared)
    DBUG_RETURN(0);
  prepared= 1;
  res= 0;
  found_rows_for_union= 0;
  TMP_TABLE_PARAM tmp_table_param;
  result= sel_result;
  t_and_f= tables_and_fields_initied;

  bzero((char *)&tmp_table_param,sizeof(TMP_TABLE_PARAM));
  thd->lex.current_select= select_cursor= first_select_in_union();
  /* Global option */
  if (((void*)(global_parameters)) == ((void*)this))
  {
    found_rows_for_union= first_select()->options & OPTION_FOUND_ROWS &&
      global_parameters->select_limit;
    if (found_rows_for_union)
      first_select()->options ^=  OPTION_FOUND_ROWS;
  }
  if (t_and_f)
  {
    // Item list and tables will be initialized by mysql_derived
    item_list= select_cursor->item_list;
  }
  else
  {
    item_list.empty();
    TABLE_LIST *first_table= (TABLE_LIST*) select_cursor->table_list.first;

    if (setup_tables(first_table) ||
	setup_wild(thd, first_table, select_cursor->item_list, 0,
		   select_cursor->with_wild))
      goto err;
    List_iterator<Item> it(select_cursor->item_list);
    Item *item;
    while((item=it++))
      item->maybe_null=1;
    item_list= select_cursor->item_list;
    select_cursor->with_wild= 0;
    if (setup_ref_array(thd, &select_cursor->ref_pointer_array,
			(item_list.elements + select_cursor->with_sum_func +
			 select_cursor->order_list.elements +
			 select_cursor->group_list.elements)) ||
	setup_fields(thd, select_cursor->ref_pointer_array, first_table,
		     item_list, 0, 0, 1))
      goto err;
    t_and_f= 1;
  }

  tmp_table_param.field_count=item_list.elements;
  if (!(table= create_tmp_table(thd, &tmp_table_param, item_list,
				(ORDER*) 0, !union_option,
				1, (select_cursor->options | thd->options |
				    TMP_TABLE_ALL_COLUMNS),
				HA_POS_ERROR)))
    goto err;
  table->file->extra(HA_EXTRA_WRITE_CACHE);
  table->file->extra(HA_EXTRA_IGNORE_DUP_KEY);
  bzero((char*) &result_table_list,sizeof(result_table_list));
  result_table_list.db= (char*) "";
  result_table_list.real_name=result_table_list.alias= (char*) "union";
  result_table_list.table=table;

  if (!(union_result=new select_union(table)))
    goto err;

  union_result->not_describe=1;
  union_result->tmp_table_param=tmp_table_param;

  /*
    The following piece of code is placed here solely for the purpose of
    getting correct results with EXPLAIN when UNION is withing a sub-select
    or derived table ...
  */

  if (thd->lex.describe)
  {
    for (SELECT_LEX *sl= select_cursor; sl; sl= sl->next_select())
    {
      JOIN *join= new JOIN(thd, sl->item_list,
			   sl->options | thd->options | SELECT_NO_UNLOCK,
			   union_result);
      thd->lex.current_select= sl;
      offset_limit_cnt= sl->offset_limit;
      select_limit_cnt= sl->select_limit+sl->offset_limit;
      if (select_limit_cnt < sl->select_limit)
	select_limit_cnt= HA_POS_ERROR;		// no limit
      if (select_limit_cnt == HA_POS_ERROR)
	sl->options&= ~OPTION_FOUND_ROWS;

      res= join->prepare(&sl->ref_pointer_array,
			 (TABLE_LIST*) sl->table_list.first, sl->with_wild,
			 sl->where,
			 ((sl->braces) ? sl->order_list.elements : 0) +
			 sl->group_list.elements,
			 (sl->braces) ?
			 (ORDER *)sl->order_list.first : (ORDER *) 0,
			 (ORDER*) sl->group_list.first,
			 sl->having,
			 (ORDER*) NULL,
			 sl, this, t_and_f);
      t_and_f= 0;
      if (res || thd->is_fatal_error)
	goto err;
    }
  }

  item_list.empty();
  thd->lex.current_select= lex_select_save;
  {
    List_iterator<Item> it(select_cursor->item_list);
    Field **field;

    for (field= table->field; *field; field++)
    {
      (void) it++;
      if (item_list.push_back(new Item_field(*field)))
	DBUG_RETURN(-1);
    }
  }

  DBUG_RETURN(res || thd->is_fatal_error ? 1 : 0);
err:
  thd->lex.current_select= lex_select_save;
  DBUG_RETURN(-1);
}

int st_select_lex_unit::exec()
{
  int do_print_slow= 0;
  SELECT_LEX_NODE *lex_select_save= thd->lex.current_select;
  SELECT_LEX *select_cursor=first_select_in_union();
  DBUG_ENTER("st_select_lex_unit::exec");

  if (executed && !(dependent || uncacheable))
    DBUG_RETURN(0);
  executed= 1;

  if ((dependent || uncacheable) || !item || !item->assigned())
  {
    if (optimized && item && item->assigned())
    {
      item->assigned(0); // We will reinit & rexecute unit
      item->reset();
      table->file->delete_all_rows();
    }
    for (SELECT_LEX *sl= select_cursor; sl; sl= sl->next_select())
    {
      if (optimized)
	res= sl->join->reinit();
      else
      {
	JOIN *join= new JOIN(thd, sl->item_list,
			     sl->options | thd->options | SELECT_NO_UNLOCK,
			     union_result);
	thd->lex.current_select= sl;
	offset_limit_cnt= sl->offset_limit;
	select_limit_cnt= sl->select_limit+sl->offset_limit;
	if (select_limit_cnt < sl->select_limit)
	  select_limit_cnt= HA_POS_ERROR;		// no limit
	if (select_limit_cnt == HA_POS_ERROR)
	  sl->options&= ~OPTION_FOUND_ROWS;

	res= join->prepare(&sl->ref_pointer_array,
			   (TABLE_LIST*) sl->table_list.first, sl->with_wild,
			   sl->where,
			   ((sl->braces) ? sl->order_list.elements : 0) +
			   sl->group_list.elements,
			   (sl->braces) ?
			   (ORDER *)sl->order_list.first : (ORDER *) 0,
			   (ORDER*) sl->group_list.first,
			   sl->having,
			   (ORDER*) NULL,
			   sl, this, t_and_f);
	t_and_f=0;
	if (res | thd->is_fatal_error)
	{
	  thd->lex.current_select= lex_select_save;
	  DBUG_RETURN(res);
	}
	res= sl->join->optimize();
      }
      if (!res)
      {
	sl->join->exec();
	res= sl->join->error;
	if (!res && union_result->flush())
	{
	  thd->lex.current_select= lex_select_save;
	  DBUG_RETURN(1);
	}
      }
      if (res)
      {
	thd->lex.current_select= lex_select_save;
	DBUG_RETURN(res);
      }
      do_print_slow|= select_cursor->options;
    }
  }
  optimized= 1;

  /* Send result to 'result' */

  // to correct ORDER BY reference resolving
  thd->lex.current_select= select_cursor;
  res= -1;
  {
    List<Item_func_match> empty_list;
    empty_list.empty();

    if (!thd->is_fatal_error)			// Check if EOM
    {
      SELECT_LEX *fake_select  = new SELECT_LEX(&thd->lex);
      offset_limit_cnt= (select_cursor->braces ?
			 global_parameters->offset_limit : 0);
      select_limit_cnt= (select_cursor->braces ?
			 global_parameters->select_limit+
			 global_parameters->offset_limit : HA_POS_ERROR);
      if (select_limit_cnt < global_parameters->select_limit)
	select_limit_cnt= HA_POS_ERROR;		// no limit
      if (select_limit_cnt == HA_POS_ERROR)
	thd->options&= ~OPTION_FOUND_ROWS;
      fake_select->ftfunc_list= &empty_list;

      res= mysql_select(thd, &ref_pointer_array, &result_table_list,
			0, item_list, NULL,
			global_parameters->order_list.elements,
			(ORDER*)global_parameters->order_list.first,
			(ORDER*) NULL, NULL, (ORDER*) NULL,
			thd->options, result, this, fake_select, 0);
      if (found_rows_for_union && !res)
	thd->limit_found_rows = (ulonglong)table->file->records;
      fake_select->exclude();
      delete fake_select;
      /*
	Mark for slow query log if any of the union parts didn't use
	indexes efficiently
      */
      select_cursor->options= ((select_cursor->options &
				~(QUERY_NO_INDEX_USED |
				  QUERY_NO_GOOD_INDEX_USED)) |
			       do_print_slow &
			       (QUERY_NO_INDEX_USED |
				QUERY_NO_GOOD_INDEX_USED));
    }
  }
  thd->lex.current_select= lex_select_save;
  DBUG_RETURN(res);
}

int st_select_lex_unit::cleanup()
{
  int error= 0;
  DBUG_ENTER("st_select_lex_unit::cleanup");

  if (union_result)
  {
    delete union_result;
    if (table)
      free_tmp_table(thd, table);
    table= 0; // Safety
  }
  for (SELECT_LEX *sl= first_select_in_union(); sl; sl= sl->next_select())
  {
    JOIN *join;
    if ((join= sl->join))
    {
      error|= sl->join->cleanup();
      delete join;
    }
  }
  DBUG_RETURN(error);
}
<commit_msg>code cleanup<commit_after>/* Copyright (C) 2000-2003 MySQL AB

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */

/*
  UNION  of select's
  UNION's  were introduced by Monty and Sinisa <sinisa@mysql.com>
*/

#include "mysql_priv.h"
#include "sql_select.h"

int mysql_union(THD *thd, LEX *lex, select_result *result,
		SELECT_LEX_UNIT *unit, bool tables_and_fields_initied)
{
  DBUG_ENTER("mysql_union");
  int res= 0;
  if (!(res= unit->prepare(thd, result, tables_and_fields_initied)))
    res= unit->exec();
  res|= unit->cleanup();
  DBUG_RETURN(res);
}

/***************************************************************************
** store records in temporary table for UNION
***************************************************************************/

select_union::select_union(TABLE *table_par)
  :table(table_par), not_describe(0)
{
  bzero((char*) &info,sizeof(info));
  /*
    We can always use DUP_IGNORE because the temporary table will only
    contain a unique key if we are using not using UNION ALL
  */
  info.handle_duplicates= DUP_IGNORE;
}

select_union::~select_union()
{
}

int select_union::prepare(List<Item> &list, SELECT_LEX_UNIT *u)
{
  unit= u;
  if (not_describe && list.elements != table->fields)
  {
    my_message(ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT,
	       ER(ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT),MYF(0));
    return -1;
  }
  return 0;
}

bool select_union::send_data(List<Item> &values)
{
  if (unit->offset_limit_cnt)
  {						// using limit offset,count
    unit->offset_limit_cnt--;
    return 0;
  }
  fill_record(table->field, values, 1);
  if (thd->net.report_error || write_record(table,&info))
  {
    if (thd->net.last_errno == ER_RECORD_FILE_FULL)
    {
      thd->clear_error(); // do not report user about table overflow
      if (create_myisam_from_heap(thd, table, &tmp_table_param,
				  info.last_errno, 0))
	return 1;
    }
    else
      return 1;
  }
  return 0;
}

bool select_union::send_eof()
{
  return 0;
}

bool select_union::flush()
{
  int error;
  if ((error=table->file->extra(HA_EXTRA_NO_CACHE)))
  {
    table->file->print_error(error,MYF(0));
    ::send_error(thd);
    return 1;
  }
  return 0;
}

int st_select_lex_unit::prepare(THD *thd, select_result *sel_result,
				bool tables_and_fields_initied)
{
  SELECT_LEX_NODE *lex_select_save= thd->lex.current_select;
  SELECT_LEX *select_cursor;
  DBUG_ENTER("st_select_lex_unit::prepare");

  if (prepared)
    DBUG_RETURN(0);
  prepared= 1;
  res= 0;
  found_rows_for_union= 0;
  TMP_TABLE_PARAM tmp_table_param;
  result= sel_result;
  t_and_f= tables_and_fields_initied;

  bzero((char *)&tmp_table_param,sizeof(TMP_TABLE_PARAM));
  thd->lex.current_select= select_cursor= first_select_in_union();
  /* Global option */
  if (((void*)(global_parameters)) == ((void*)this))
  {
    found_rows_for_union= first_select()->options & OPTION_FOUND_ROWS &&
      global_parameters->select_limit;
    if (found_rows_for_union)
      first_select()->options&=  ~OPTION_FOUND_ROWS;
  }
  if (t_and_f)
  {
    // Item list and tables will be initialized by mysql_derived
    item_list= select_cursor->item_list;
  }
  else
  {
    item_list.empty();
    TABLE_LIST *first_table= (TABLE_LIST*) select_cursor->table_list.first;

    if (setup_tables(first_table) ||
	setup_wild(thd, first_table, select_cursor->item_list, 0,
		   select_cursor->with_wild))
      goto err;
    List_iterator<Item> it(select_cursor->item_list);
    Item *item;
    while((item=it++))
      item->maybe_null=1;
    item_list= select_cursor->item_list;
    select_cursor->with_wild= 0;
    if (setup_ref_array(thd, &select_cursor->ref_pointer_array,
			(item_list.elements + select_cursor->with_sum_func +
			 select_cursor->order_list.elements +
			 select_cursor->group_list.elements)) ||
	setup_fields(thd, select_cursor->ref_pointer_array, first_table,
		     item_list, 0, 0, 1))
      goto err;
    t_and_f= 1;
  }

  tmp_table_param.field_count=item_list.elements;
  if (!(table= create_tmp_table(thd, &tmp_table_param, item_list,
				(ORDER*) 0, !union_option,
				1, (select_cursor->options | thd->options |
				    TMP_TABLE_ALL_COLUMNS),
				HA_POS_ERROR)))
    goto err;
  table->file->extra(HA_EXTRA_WRITE_CACHE);
  table->file->extra(HA_EXTRA_IGNORE_DUP_KEY);
  bzero((char*) &result_table_list,sizeof(result_table_list));
  result_table_list.db= (char*) "";
  result_table_list.real_name=result_table_list.alias= (char*) "union";
  result_table_list.table=table;

  if (!(union_result=new select_union(table)))
    goto err;

  union_result->not_describe=1;
  union_result->tmp_table_param=tmp_table_param;

  /*
    The following piece of code is placed here solely for the purpose of
    getting correct results with EXPLAIN when UNION is withing a sub-select
    or derived table ...
  */

  if (thd->lex.describe)
  {
    for (SELECT_LEX *sl= select_cursor; sl; sl= sl->next_select())
    {
      JOIN *join= new JOIN(thd, sl->item_list,
			   sl->options | thd->options | SELECT_NO_UNLOCK,
			   union_result);
      thd->lex.current_select= sl;
      offset_limit_cnt= sl->offset_limit;
      select_limit_cnt= sl->select_limit+sl->offset_limit;
      if (select_limit_cnt < sl->select_limit)
	select_limit_cnt= HA_POS_ERROR;		// no limit
      if (select_limit_cnt == HA_POS_ERROR)
	sl->options&= ~OPTION_FOUND_ROWS;

      res= join->prepare(&sl->ref_pointer_array,
			 (TABLE_LIST*) sl->table_list.first, sl->with_wild,
			 sl->where,
			 ((sl->braces) ? sl->order_list.elements : 0) +
			 sl->group_list.elements,
			 (sl->braces) ?
			 (ORDER *)sl->order_list.first : (ORDER *) 0,
			 (ORDER*) sl->group_list.first,
			 sl->having,
			 (ORDER*) NULL,
			 sl, this, t_and_f);
      t_and_f= 0;
      if (res || thd->is_fatal_error)
	goto err;
    }
  }

  item_list.empty();
  thd->lex.current_select= lex_select_save;
  {
    List_iterator<Item> it(select_cursor->item_list);
    Field **field;

    for (field= table->field; *field; field++)
    {
      (void) it++;
      if (item_list.push_back(new Item_field(*field)))
	DBUG_RETURN(-1);
    }
  }

  DBUG_RETURN(res || thd->is_fatal_error ? 1 : 0);
err:
  thd->lex.current_select= lex_select_save;
  DBUG_RETURN(-1);
}

int st_select_lex_unit::exec()
{
  int do_print_slow= 0;
  SELECT_LEX_NODE *lex_select_save= thd->lex.current_select;
  SELECT_LEX *select_cursor=first_select_in_union();
  DBUG_ENTER("st_select_lex_unit::exec");

  if (executed && !(dependent || uncacheable))
    DBUG_RETURN(0);
  executed= 1;

  if ((dependent || uncacheable) || !item || !item->assigned())
  {
    if (optimized && item && item->assigned())
    {
      item->assigned(0); // We will reinit & rexecute unit
      item->reset();
      table->file->delete_all_rows();
    }
    for (SELECT_LEX *sl= select_cursor; sl; sl= sl->next_select())
    {
      if (optimized)
	res= sl->join->reinit();
      else
      {
	JOIN *join= new JOIN(thd, sl->item_list,
			     sl->options | thd->options | SELECT_NO_UNLOCK,
			     union_result);
	thd->lex.current_select= sl;
	offset_limit_cnt= sl->offset_limit;
	select_limit_cnt= sl->select_limit+sl->offset_limit;
	if (select_limit_cnt < sl->select_limit)
	  select_limit_cnt= HA_POS_ERROR;		// no limit
	if (select_limit_cnt == HA_POS_ERROR)
	  sl->options&= ~OPTION_FOUND_ROWS;

	res= join->prepare(&sl->ref_pointer_array,
			   (TABLE_LIST*) sl->table_list.first, sl->with_wild,
			   sl->where,
			   ((sl->braces) ? sl->order_list.elements : 0) +
			   sl->group_list.elements,
			   (sl->braces) ?
			   (ORDER *)sl->order_list.first : (ORDER *) 0,
			   (ORDER*) sl->group_list.first,
			   sl->having,
			   (ORDER*) NULL,
			   sl, this, t_and_f);
	t_and_f=0;
	if (res | thd->is_fatal_error)
	{
	  thd->lex.current_select= lex_select_save;
	  DBUG_RETURN(res);
	}
	res= sl->join->optimize();
      }
      if (!res)
      {
	sl->join->exec();
	res= sl->join->error;
	if (!res && union_result->flush())
	{
	  thd->lex.current_select= lex_select_save;
	  DBUG_RETURN(1);
	}
      }
      if (res)
      {
	thd->lex.current_select= lex_select_save;
	DBUG_RETURN(res);
      }
      do_print_slow|= select_cursor->options;
    }
  }
  optimized= 1;

  /* Send result to 'result' */

  // to correct ORDER BY reference resolving
  thd->lex.current_select= select_cursor;
  res= -1;
  {
    List<Item_func_match> empty_list;
    empty_list.empty();

    if (!thd->is_fatal_error)			// Check if EOM
    {
      SELECT_LEX *fake_select  = new SELECT_LEX(&thd->lex);
      offset_limit_cnt= (select_cursor->braces ?
			 global_parameters->offset_limit : 0);
      select_limit_cnt= (select_cursor->braces ?
			 global_parameters->select_limit+
			 global_parameters->offset_limit : HA_POS_ERROR);
      if (select_limit_cnt < global_parameters->select_limit)
	select_limit_cnt= HA_POS_ERROR;		// no limit
      if (select_limit_cnt == HA_POS_ERROR)
	thd->options&= ~OPTION_FOUND_ROWS;
      fake_select->ftfunc_list= &empty_list;

      res= mysql_select(thd, &ref_pointer_array, &result_table_list,
			0, item_list, NULL,
			global_parameters->order_list.elements,
			(ORDER*)global_parameters->order_list.first,
			(ORDER*) NULL, NULL, (ORDER*) NULL,
			thd->options, result, this, fake_select, 0);
      if (found_rows_for_union && !res)
	thd->limit_found_rows = (ulonglong)table->file->records;
      fake_select->exclude();
      delete fake_select;
      /*
	Mark for slow query log if any of the union parts didn't use
	indexes efficiently
      */
      select_cursor->options= ((select_cursor->options &
				~(QUERY_NO_INDEX_USED |
				  QUERY_NO_GOOD_INDEX_USED)) |
			       do_print_slow &
			       (QUERY_NO_INDEX_USED |
				QUERY_NO_GOOD_INDEX_USED));
    }
  }
  thd->lex.current_select= lex_select_save;
  DBUG_RETURN(res);
}

int st_select_lex_unit::cleanup()
{
  int error= 0;
  DBUG_ENTER("st_select_lex_unit::cleanup");

  if (union_result)
  {
    delete union_result;
    if (table)
      free_tmp_table(thd, table);
    table= 0; // Safety
  }
  for (SELECT_LEX *sl= first_select_in_union(); sl; sl= sl->next_select())
  {
    JOIN *join;
    if ((join= sl->join))
    {
      error|= sl->join->cleanup();
      delete join;
    }
  }
  DBUG_RETURN(error);
}
<|endoftext|>


<commit_before>/*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "webrtc/modules/utility/source/process_thread_impl.h"

#include "webrtc/base/checks.h"
#include "webrtc/modules/interface/module.h"
#include "webrtc/system_wrappers/interface/logging.h"
#include "webrtc/system_wrappers/interface/tick_util.h"

namespace webrtc {
namespace {

// We use this constant internally to signal that a module has requested
// a callback right away.  When this is set, no call to TimeUntilNextProcess
// should be made, but Process() should be called directly.
const int64_t kCallProcessImmediately = -1;

int64_t GetNextCallbackTime(Module* module, int64_t time_now) {
  int64_t interval = module->TimeUntilNextProcess();
  // Currently some implementations erroneously return error codes from
  // TimeUntilNextProcess(). So, as is, we correct that and log an error.
  if (interval < 0) {
    LOG(LS_ERROR) << "TimeUntilNextProcess returned an invalid value "
                  << interval;
    interval = 0;
  }
  return time_now + interval;
}
}

ProcessThread::~ProcessThread() {}

// static
rtc::scoped_ptr<ProcessThread> ProcessThread::Create() {
  return rtc::scoped_ptr<ProcessThread>(new ProcessThreadImpl()).Pass();
}

ProcessThreadImpl::ProcessThreadImpl()
    : wake_up_(EventWrapper::Create()), stop_(false) {
}

ProcessThreadImpl::~ProcessThreadImpl() {
  DCHECK(thread_checker_.CalledOnValidThread());
  DCHECK(!thread_.get());
  DCHECK(!stop_);

  while (!queue_.empty()) {
    delete queue_.front();
    queue_.pop();
  }
}

void ProcessThreadImpl::Start() {
  DCHECK(thread_checker_.CalledOnValidThread());
  DCHECK(!thread_.get());
  if (thread_.get())
    return;

  DCHECK(!stop_);

  {
    // TODO(tommi): Since DeRegisterModule is currently being called from
    // different threads in some cases (ChannelOwner), we need to lock access to
    // the modules_ collection even on the controller thread.
    // Once we've cleaned up those places, we can remove this lock.
    rtc::CritScope lock(&lock_);
    for (ModuleCallback& m : modules_)
      m.module->ProcessThreadAttached(this);
  }

  thread_ = ThreadWrapper::CreateThread(
      &ProcessThreadImpl::Run, this, "ProcessThread");
  CHECK(thread_->Start());
}

void ProcessThreadImpl::Stop() {
  DCHECK(thread_checker_.CalledOnValidThread());
  if(!thread_.get())
    return;

  {
    rtc::CritScope lock(&lock_);
    stop_ = true;
  }

  wake_up_->Set();

  CHECK(thread_->Stop());
  stop_ = false;

  // TODO(tommi): Since DeRegisterModule is currently being called from
  // different threads in some cases (ChannelOwner), we need to lock access to
  // the modules_ collection even on the controller thread.
  // Since DeRegisterModule also checks thread_, we also need to hold the
  // lock for the .reset() operation.
  // Once we've cleaned up those places, we can remove this lock.
  rtc::CritScope lock(&lock_);
  thread_.reset();
  for (ModuleCallback& m : modules_)
    m.module->ProcessThreadAttached(nullptr);
}

void ProcessThreadImpl::WakeUp(Module* module) {
  // Allowed to be called on any thread.
  {
    rtc::CritScope lock(&lock_);
    for (ModuleCallback& m : modules_) {
      if (m.module == module)
        m.next_callback = kCallProcessImmediately;
    }
  }
  wake_up_->Set();
}

void ProcessThreadImpl::PostTask(rtc::scoped_ptr<ProcessTask> task) {
  // Allowed to be called on any thread.
  {
    rtc::CritScope lock(&lock_);
    queue_.push(task.release());
  }
  wake_up_->Set();
}

void ProcessThreadImpl::RegisterModule(Module* module) {
  DCHECK(thread_checker_.CalledOnValidThread());
  DCHECK(module);

#if (!defined(NDEBUG) || defined(DCHECK_ALWAYS_ON))
  {
    // Catch programmer error.
    rtc::CritScope lock(&lock_);
    for (const ModuleCallback& mc : modules_)
      DCHECK(mc.module != module);
  }
#endif

  // Now that we know the module isn't in the list, we'll call out to notify
  // the module that it's attached to the worker thread.  We don't hold
  // the lock while we make this call.
  if (thread_.get())
    module->ProcessThreadAttached(this);

  {
    rtc::CritScope lock(&lock_);
    modules_.push_back(ModuleCallback(module));
  }

  // Wake the thread calling ProcessThreadImpl::Process() to update the
  // waiting time. The waiting time for the just registered module may be
  // shorter than all other registered modules.
  wake_up_->Set();
}

void ProcessThreadImpl::DeRegisterModule(Module* module) {
  // Allowed to be called on any thread.
  // TODO(tommi): Disallow this ^^^
  DCHECK(module);

  {
    rtc::CritScope lock(&lock_);
    modules_.remove_if([&module](const ModuleCallback& m) {
        return m.module == module;
      });

    // TODO(tommi): we currently need to hold the lock while calling out to
    // ProcessThreadAttached.  This is to make sure that the thread hasn't been
    // destroyed while we attach the module.  Once we can make sure
    // DeRegisterModule isn't being called on arbitrary threads, we can move the
    // |if (thread_.get())| check and ProcessThreadAttached() call outside the
    // lock scope.

    // Notify the module that it's been detached.
    if (thread_.get())
      module->ProcessThreadAttached(nullptr);
  }
}

// static
bool ProcessThreadImpl::Run(void* obj) {
  return static_cast<ProcessThreadImpl*>(obj)->Process();
}

bool ProcessThreadImpl::Process() {
  int64_t now = TickTime::MillisecondTimestamp();
  int64_t next_checkpoint = now + (1000 * 60);

  {
    rtc::CritScope lock(&lock_);
    if (stop_)
      return false;
    for (ModuleCallback& m : modules_) {
      // TODO(tommi): Would be good to measure the time TimeUntilNextProcess
      // takes and dcheck if it takes too long (e.g. >=10ms).  Ideally this
      // operation should not require taking a lock, so querying all modules
      // should run in a matter of nanoseconds.
      if (m.next_callback == 0)
        m.next_callback = GetNextCallbackTime(m.module, now);

      if (m.next_callback <= now ||
          m.next_callback == kCallProcessImmediately) {
        m.module->Process();
        // Use a new 'now' reference to calculate when the next callback
        // should occur.  We'll continue to use 'now' above for the baseline
        // of calculating how long we should wait, to reduce variance.
        int64_t new_now = TickTime::MillisecondTimestamp();
        m.next_callback = GetNextCallbackTime(m.module, new_now);
      }

      if (m.next_callback < next_checkpoint)
        next_checkpoint = m.next_callback;
    }

    while (!queue_.empty()) {
      ProcessTask* task = queue_.front();
      queue_.pop();
      lock_.Leave();
      task->Run();
      delete task;
      lock_.Enter();
    }
  }

  int64_t time_to_wait = next_checkpoint - TickTime::MillisecondTimestamp();
  if (time_to_wait > 0)
    wake_up_->Wait(static_cast<unsigned long>(time_to_wait));

  return true;
}
}  // namespace webrtc
<commit_msg>Prevent TimeUntilNextProcess log spam.<commit_after>/*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "webrtc/modules/utility/source/process_thread_impl.h"

#include "webrtc/base/checks.h"
#include "webrtc/modules/interface/module.h"
#include "webrtc/system_wrappers/interface/logging.h"
#include "webrtc/system_wrappers/interface/tick_util.h"

namespace webrtc {
namespace {

// We use this constant internally to signal that a module has requested
// a callback right away.  When this is set, no call to TimeUntilNextProcess
// should be made, but Process() should be called directly.
const int64_t kCallProcessImmediately = -1;

int64_t GetNextCallbackTime(Module* module, int64_t time_now) {
  int64_t interval = module->TimeUntilNextProcess();
  if (interval < 0) {
    // Falling behind, we should call the callback now.
    return time_now;
  }
  return time_now + interval;
}
}

ProcessThread::~ProcessThread() {}

// static
rtc::scoped_ptr<ProcessThread> ProcessThread::Create() {
  return rtc::scoped_ptr<ProcessThread>(new ProcessThreadImpl()).Pass();
}

ProcessThreadImpl::ProcessThreadImpl()
    : wake_up_(EventWrapper::Create()), stop_(false) {
}

ProcessThreadImpl::~ProcessThreadImpl() {
  DCHECK(thread_checker_.CalledOnValidThread());
  DCHECK(!thread_.get());
  DCHECK(!stop_);

  while (!queue_.empty()) {
    delete queue_.front();
    queue_.pop();
  }
}

void ProcessThreadImpl::Start() {
  DCHECK(thread_checker_.CalledOnValidThread());
  DCHECK(!thread_.get());
  if (thread_.get())
    return;

  DCHECK(!stop_);

  {
    // TODO(tommi): Since DeRegisterModule is currently being called from
    // different threads in some cases (ChannelOwner), we need to lock access to
    // the modules_ collection even on the controller thread.
    // Once we've cleaned up those places, we can remove this lock.
    rtc::CritScope lock(&lock_);
    for (ModuleCallback& m : modules_)
      m.module->ProcessThreadAttached(this);
  }

  thread_ = ThreadWrapper::CreateThread(
      &ProcessThreadImpl::Run, this, "ProcessThread");
  CHECK(thread_->Start());
}

void ProcessThreadImpl::Stop() {
  DCHECK(thread_checker_.CalledOnValidThread());
  if(!thread_.get())
    return;

  {
    rtc::CritScope lock(&lock_);
    stop_ = true;
  }

  wake_up_->Set();

  CHECK(thread_->Stop());
  stop_ = false;

  // TODO(tommi): Since DeRegisterModule is currently being called from
  // different threads in some cases (ChannelOwner), we need to lock access to
  // the modules_ collection even on the controller thread.
  // Since DeRegisterModule also checks thread_, we also need to hold the
  // lock for the .reset() operation.
  // Once we've cleaned up those places, we can remove this lock.
  rtc::CritScope lock(&lock_);
  thread_.reset();
  for (ModuleCallback& m : modules_)
    m.module->ProcessThreadAttached(nullptr);
}

void ProcessThreadImpl::WakeUp(Module* module) {
  // Allowed to be called on any thread.
  {
    rtc::CritScope lock(&lock_);
    for (ModuleCallback& m : modules_) {
      if (m.module == module)
        m.next_callback = kCallProcessImmediately;
    }
  }
  wake_up_->Set();
}

void ProcessThreadImpl::PostTask(rtc::scoped_ptr<ProcessTask> task) {
  // Allowed to be called on any thread.
  {
    rtc::CritScope lock(&lock_);
    queue_.push(task.release());
  }
  wake_up_->Set();
}

void ProcessThreadImpl::RegisterModule(Module* module) {
  DCHECK(thread_checker_.CalledOnValidThread());
  DCHECK(module);

#if (!defined(NDEBUG) || defined(DCHECK_ALWAYS_ON))
  {
    // Catch programmer error.
    rtc::CritScope lock(&lock_);
    for (const ModuleCallback& mc : modules_)
      DCHECK(mc.module != module);
  }
#endif

  // Now that we know the module isn't in the list, we'll call out to notify
  // the module that it's attached to the worker thread.  We don't hold
  // the lock while we make this call.
  if (thread_.get())
    module->ProcessThreadAttached(this);

  {
    rtc::CritScope lock(&lock_);
    modules_.push_back(ModuleCallback(module));
  }

  // Wake the thread calling ProcessThreadImpl::Process() to update the
  // waiting time. The waiting time for the just registered module may be
  // shorter than all other registered modules.
  wake_up_->Set();
}

void ProcessThreadImpl::DeRegisterModule(Module* module) {
  // Allowed to be called on any thread.
  // TODO(tommi): Disallow this ^^^
  DCHECK(module);

  {
    rtc::CritScope lock(&lock_);
    modules_.remove_if([&module](const ModuleCallback& m) {
        return m.module == module;
      });

    // TODO(tommi): we currently need to hold the lock while calling out to
    // ProcessThreadAttached.  This is to make sure that the thread hasn't been
    // destroyed while we attach the module.  Once we can make sure
    // DeRegisterModule isn't being called on arbitrary threads, we can move the
    // |if (thread_.get())| check and ProcessThreadAttached() call outside the
    // lock scope.

    // Notify the module that it's been detached.
    if (thread_.get())
      module->ProcessThreadAttached(nullptr);
  }
}

// static
bool ProcessThreadImpl::Run(void* obj) {
  return static_cast<ProcessThreadImpl*>(obj)->Process();
}

bool ProcessThreadImpl::Process() {
  int64_t now = TickTime::MillisecondTimestamp();
  int64_t next_checkpoint = now + (1000 * 60);

  {
    rtc::CritScope lock(&lock_);
    if (stop_)
      return false;
    for (ModuleCallback& m : modules_) {
      // TODO(tommi): Would be good to measure the time TimeUntilNextProcess
      // takes and dcheck if it takes too long (e.g. >=10ms).  Ideally this
      // operation should not require taking a lock, so querying all modules
      // should run in a matter of nanoseconds.
      if (m.next_callback == 0)
        m.next_callback = GetNextCallbackTime(m.module, now);

      if (m.next_callback <= now ||
          m.next_callback == kCallProcessImmediately) {
        m.module->Process();
        // Use a new 'now' reference to calculate when the next callback
        // should occur.  We'll continue to use 'now' above for the baseline
        // of calculating how long we should wait, to reduce variance.
        int64_t new_now = TickTime::MillisecondTimestamp();
        m.next_callback = GetNextCallbackTime(m.module, new_now);
      }

      if (m.next_callback < next_checkpoint)
        next_checkpoint = m.next_callback;
    }

    while (!queue_.empty()) {
      ProcessTask* task = queue_.front();
      queue_.pop();
      lock_.Leave();
      task->Run();
      delete task;
      lock_.Enter();
    }
  }

  int64_t time_to_wait = next_checkpoint - TickTime::MillisecondTimestamp();
  if (time_to_wait > 0)
    wake_up_->Wait(static_cast<unsigned long>(time_to_wait));

  return true;
}
}  // namespace webrtc
<|endoftext|>


<commit_before>/*!
@file

@copyright Edouard Alligand and Joel Falcou 2015-2017
(See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
*/
#ifndef BOOST_BRIGAND_ALGORITHMS_ALL_HPP
#define BOOST_BRIGAND_ALGORITHMS_ALL_HPP

#include <brigand/algorithms/detail/non_null.hpp>
#include <brigand/config.hpp>
#include <brigand/functions/lambda/apply.hpp>
#include <initializer_list>
#include <type_traits>

namespace brigand
{
#if defined(BRIGAND_COMP_MSVC_2013) || defined(BRIGAND_COMP_CUDA) || defined(BRIGAND_COMP_INTEL)
namespace detail
{
    template <class P, class T>
    struct all_helper : ::brigand::apply<P, T>
    {
    };

    template <bool...>
    struct bools_
    {
    };
    template <typename Sequence, typename Predicate, typename... Ts>
    struct all_impl;

    template <template <class...> class Sequence, typename Predicate, typename... Ts>
    struct all_impl<Sequence<Ts...>, Predicate>
        : std::is_same<bools_<true, all_helper<Predicate, Ts>::value...>,
                       bools_<all_helper<Predicate, Ts>::value..., true>>
    {
    };
}
#else
namespace detail
{
    struct all_same
    {
        const bool value = false;
        constexpr all_same(...) {}
        template <typename T>
        constexpr all_same(std::initializer_list<T *>) : value{true}
        {
        }
    };

    template <typename Sequence, typename Predicate>
    struct all_impl : bool_<true>
    {
    };

    template <template <class...> class Sequence, typename Predicate, typename T, typename... Ts>
    struct all_impl<Sequence<T, Ts...>, Predicate>
    {
        static constexpr all_same tester{
            static_cast< ::brigand::apply<Predicate, T> *>(nullptr),
            static_cast< ::brigand::apply<Predicate, Ts> *>(nullptr)...};
        using type = bool_<(::brigand::apply<Predicate, T>::value != 0 && tester.value)>;
    };

    template <template <class...> class Sequence, template <typename...> class F, typename T,
              typename... Ts>
    struct all_impl<Sequence<T, Ts...>, bind<F, _1>>
    {
        static constexpr all_same tester{static_cast<F<T> *>(nullptr),
                                         static_cast<F<Ts> *>(nullptr)...};
        using type = bool_<(F<T>::value != 0 && tester.value)>;
    };

    template <template <class...> class Sequence, template <typename...> class F, typename T,
              typename... Ts>
    struct all_impl<Sequence<T, Ts...>, F<_1>>
    {
        static constexpr all_same tester{static_cast<typename F<T>::type *>(nullptr),
                                         static_cast<typename F<Ts>::type *>(nullptr)...};
        using type = bool_<(F<T>::type::value != 0 && tester.value)>;
    };
}
#endif

// Is a predicate true for every type ?
template <typename Sequence, typename Predicate = detail::non_null>
using all = typename detail::all_impl<Sequence, Predicate>::type;
}
#endif
<commit_msg>Delete all.hpp<commit_after><|endoftext|>


<commit_before>// Copyright (c) 2018 alex v
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include "ntpclient.h"
#include "util.h"

using namespace boost;
using namespace boost::asio;

int64_t CNtpClient::getTimestamp()
{
    time_t timeRecv = -1;

    io_service io_service;

    LogPrint("ntp", "CNtpClient: Opening socket to NTP server %s.\n", sHostName);
    try
    {

        ip::udp::resolver resolver(io_service);
        ip::udp::resolver::query query(boost::asio::ip::udp::v4(), sHostName, "ntp");

        ip::udp::endpoint receiver_endpoint = *resolver.resolve(query);

        ip::udp::socket socket(io_service);
        socket.open(ip::udp::v4());

        boost::array<unsigned char, 48> sendBuf = {10,0,0,0,0,0,0,0,0};

        socket.send_to(boost::asio::buffer(sendBuf), receiver_endpoint);

        boost::array<unsigned long, 1024> recvBuf;
        ip::udp::endpoint sender_endpoint;

        try
        {

            fd_set fileDescriptorSet;
            struct timeval timeStruct;

            // set the timeout to 5 seconds
            timeStruct.tv_sec = GetArg("-ntptimeout", 5);
            timeStruct.tv_usec = 0;
            FD_ZERO(&fileDescriptorSet);

            int nativeSocket = socket.native();

            FD_SET(nativeSocket,&fileDescriptorSet);

            select(nativeSocket+1,&fileDescriptorSet,NULL,NULL,&timeStruct);

            if(!FD_ISSET(nativeSocket,&fileDescriptorSet))
            {

                LogPrint("ntp", "CNtpClient: Could not read socket from NTP server %s (Read timeout)\n", sHostName);

            }
            else
            {

                socket.receive_from(boost::asio::buffer(recvBuf), sender_endpoint);
                timeRecv = ntohl((time_t)recvBuf[4]);
                timeRecv-= 2208988800U;  // Substract 01/01/1970 == 2208988800U

                LogPrint("ntp", "CNtpClient: Received timestamp: %ll", (long)timeRecv);

            }

        }
        catch (std::exception& e)
        {

             LogPrintf("CNtpClient: Could not read clock from NTP server %s (%s)\n", sHostName, e.what());

        }

    }
    catch (std::exception& e)
    {

        LogPrintf("CNtpClient: Could not open socket to NTP server %s (%s)\n", sHostName, e.what());

    }

    return (int64_t)timeRecv;
  }
<commit_msg>fix definition<commit_after>// Copyright (c) 2018 alex v
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include "ntpclient.h"
#include "util.h"

using namespace boost;
using namespace boost::asio;

int64_t CNtpClient::getTimestamp()
{
    time_t timeRecv = -1;

    io_service io_service;

    LogPrint("ntp", "CNtpClient: Opening socket to NTP server %s.\n", sHostName);
    try
    {

        ip::udp::resolver resolver(io_service);
        ip::udp::resolver::query query(boost::asio::ip::udp::v4(), sHostName, "ntp");

        ip::udp::endpoint receiver_endpoint = *resolver.resolve(query);

        ip::udp::socket socket(io_service);
        socket.open(ip::udp::v4());

        boost::array<unsigned char, 48> sendBuf = {{10,0,0,0,0,0,0,0,0}};

        socket.send_to(boost::asio::buffer(sendBuf), receiver_endpoint);

        boost::array<unsigned long, 1024> recvBuf;
        ip::udp::endpoint sender_endpoint;

        try
        {

            fd_set fileDescriptorSet;
            struct timeval timeStruct;

            // set the timeout to 5 seconds
            timeStruct.tv_sec = GetArg("-ntptimeout", 5);
            timeStruct.tv_usec = 0;
            FD_ZERO(&fileDescriptorSet);

            int nativeSocket = socket.native();

            FD_SET(nativeSocket,&fileDescriptorSet);

            select(nativeSocket+1,&fileDescriptorSet,NULL,NULL,&timeStruct);

            if(!FD_ISSET(nativeSocket,&fileDescriptorSet))
            {

                LogPrint("ntp", "CNtpClient: Could not read socket from NTP server %s (Read timeout)\n", sHostName);

            }
            else
            {

                socket.receive_from(boost::asio::buffer(recvBuf), sender_endpoint);
                timeRecv = ntohl((time_t)recvBuf[4]);
                timeRecv-= 2208988800U;  // Substract 01/01/1970 == 2208988800U

                LogPrint("ntp", "CNtpClient: Received timestamp: %ll", (long)timeRecv);

            }

        }
        catch (std::exception& e)
        {

             LogPrintf("CNtpClient: Could not read clock from NTP server %s (%s)\n", sHostName, e.what());

        }

    }
    catch (std::exception& e)
    {

        LogPrintf("CNtpClient: Could not open socket to NTP server %s (%s)\n", sHostName, e.what());

    }

    return (int64_t)timeRecv;
  }
<|endoftext|>


<commit_before>//-----------------------------------------------------------------------------
// Copyright (c) 2015, Jeff Hutchinson
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright notice, this
//   list of conditions and the following disclaimer.
//
// * Redistributions in binary form must reproduce the above copyright notice,
//   this list of conditions and the following disclaimer in the documentation
//   and/or other materials provided with the distribution.
//
// * Neither the name of D3D11Test nor the names of its
//   contributors may be used to endorse or promote products derived from
//   this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//-----------------------------------------------------------------------------

// Require MSVC compiler for this application
#ifndef _MSC_VER
#error "You must use MSVC to compile this application."
#endif // _MSC_VER

#include <stdlib.h>
#include <SDL.h>

// Windows specific header + D3D11 SDK header
#include <Windows.h>
#include <d3d11.h>

// SDL2 hack.
#ifdef main
#undef main
#endif // main

int main(int argc, const char **argv) {
	SDL_Init(SDL_INIT_VIDEO);

	// create window
	SDL_Window *window = SDL_CreateWindow("D3D11 Test Application", SDL_WINDOWPOS_CENTERED, SDL_WINDOWPOS_CENTERED, 1366, 768, SDL_WINDOW_SHOWN);
	if (window == nullptr) {
		SDL_Quit();
		return 1;
	}

	SDL_Event event;
	bool running = true;
	while (running) {
		// Run event loop.
		while (SDL_PollEvent(&event)) {
			// check to see if we fired a quit event. If we did, we're done
			// running the program.
			if (event.type == SDL_QUIT) {
				running = false;
				break;
			}
		}
	}

	// cleanup
	SDL_DestroyWindow(window);
	SDL_Quit();
	return 0;
}<commit_msg>should use events as well<commit_after>//-----------------------------------------------------------------------------
// Copyright (c) 2015, Jeff Hutchinson
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright notice, this
//   list of conditions and the following disclaimer.
//
// * Redistributions in binary form must reproduce the above copyright notice,
//   this list of conditions and the following disclaimer in the documentation
//   and/or other materials provided with the distribution.
//
// * Neither the name of D3D11Test nor the names of its
//   contributors may be used to endorse or promote products derived from
//   this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//-----------------------------------------------------------------------------

// Require MSVC compiler for this application
#ifndef _MSC_VER
#error "You must use MSVC to compile this application."
#endif // _MSC_VER

#include <stdlib.h>
#include <SDL.h>

// Windows specific header + D3D11 SDK header
#include <Windows.h>
#include <d3d11.h>

// SDL2 hack.
#ifdef main
#undef main
#endif // main

int main(int argc, const char **argv) {
	SDL_Init(SDL_INIT_VIDEO | SDL_INIT_EVENTS);

	// create window
	SDL_Window *window = SDL_CreateWindow("D3D11 Test Application", SDL_WINDOWPOS_CENTERED, SDL_WINDOWPOS_CENTERED, 1366, 768, SDL_WINDOW_SHOWN);
	if (window == nullptr) {
		SDL_Quit();
		return 1;
	}

	SDL_Event event;
	bool running = true;
	while (running) {
		// Run event loop.
		while (SDL_PollEvent(&event)) {
			// check to see if we fired a quit event. If we did, we're done
			// running the program.
			if (event.type == SDL_QUIT) {
				running = false;
				break;
			}
		}
	}

	// cleanup
	SDL_DestroyWindow(window);
	SDL_Quit();
	return 0;
}<|endoftext|>


<commit_before>#include <stdio.h>
#include <math.h>
#include <cstdlib>
#include <unistd.h> // For usleep

// For shared memory management
#include <sys/mman.h>

#include <ctime> // For clock()
#include <chrono> // For clock()
#include <mpi.h>

#include <sys/wait.h> // For wait(pid)

#include <limits> // For float max

// GA MPI Headers
#include <ga-mpi/ga.h>
#include <ga-mpi/std_stream.h>
#include <ga-mpi/GARealGenome.h>
#include <ga-mpi/GARealGenome.C>

// Argos headers
#include <argos3/core/simulator/simulator.h>
#include <argos3/core/simulator/loop_functions.h>
#include <source/CPFA/CPFA_loop_functions.h>

// For timing
#include <ctime> // For clock()
#include <chrono> // For clock()

// For shared variable between child and parent process
#include  <sys/ipc.h>
#include  <sys/shm.h>

float objective(GAGenome &);
float LaunchARGoS(GAGenome &);

int mpi_tasks, mpi_rank;

int n_trials = 10; // used by the objective function
double mutation_stdev = 0.1; // Used by LaunchArgos function.

int main(int argc, char **argv)
{
  std::chrono::time_point<std::chrono::system_clock> start, end;
  start = std::chrono::system_clock::now();

  srand(time(NULL));

  double mutation_rate = 0.1;
  double crossover_rate = 0.05;
  int population_size = 50;
  int n_generations = 100;

  char c='h';
  // Handle command line arguments
  while ((c = getopt (argc, argv, "tgp:")) != -1)
    switch (c)
      {
      case 't':
	n_trials = atoi(optarg);
	break;
      case 'g':
        n_generations = atoi(optarg);
        break;
      case 'p':
        population_size = atoi(optarg);
        break;
      case 'c':
	crossover_rate = strtod(optarg, NULL);
	break;
      case 'm':
        mutation_rate = strtod(optarg, NULL);
	break;
      case 'h':
	printf("Usage: %s -p {population size} -g {number of generations} -t {number of trials} -c {crossover rate} -m {mutation rate} -s {mutation standard deviation}", argv[0]);
	break;
      case '?':
        if (optopt == 'p')
          fprintf (stderr, "Option -%c requires an argument specifying the population size.\n", optopt);
	else if (optopt == 'g')
	  fprintf (stderr, "Option -%c requires an argument specifying the number of generations.\n", optopt);
	else if (optopt == 't')
	  fprintf (stderr, "Option -%c requires an argument specifying the number of trials.\n", optopt);
	else if (optopt == 'c')
	  fprintf (stderr, "Option -%c requires an argument specifying the crossover rate.\n", optopt);
	else if (optopt == 'm')
	  fprintf (stderr, "Option -%c requires an argument specifying the mutation rate.\n", optopt);
	else if (optopt == 's')
	  fprintf (stderr, "Option -%c requires an argument specifying the mutation standard deviation.\n", optopt);
        else if (isprint (optopt))
          fprintf (stderr, "Unknown option `-%c'.\n", optopt);
        else
          fprintf (stderr,
                   "Unknown option character `\\x%x'.\n",
                   optopt);
        return 1;
      default:
	printf("Usage: %s -p {population size} -g {number of generations} -t {number of trials} -c {crossover rate} -m {mutation rate} -s {mutation standard deviation}", argv[0]);
        abort ();
      }

  float max_float = std::numeric_limits<float>::max();

  // MPI init
  MPI_Init(&argc, &argv);
  MPI_Comm_size(MPI_COMM_WORLD, &mpi_tasks);
  MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);

  char hostname[1024];
  hostname[1023] = '\0';
  gethostname(hostname, 1023);
  printf("%s:\tworker %d ready.\n", hostname, mpi_rank);

  // See if we've been given a seed to use (for testing purposes).  When you
  // specify a random seed, the evolution will be exactly the same each time
  // you use that seed number
  unsigned int seed = 0;
  for(int i=1 ; i<argc ; i++)
    if(strcmp(argv[i++],"seed") == 0)
      seed = atoi(argv[i]);

  // popsize / mpi_tasks must be an integer
  population_size = mpi_tasks * int((double)population_size/(double)mpi_tasks+0.999);

  // Define the genome
  GARealAlleleSetArray allele_array;
  allele_array.add(0,1/mutation_stdev); // Probability of switching to search
  allele_array.add(0,1/mutation_stdev); // Probability of returning to nest
  allele_array.add(0,4*M_PI/mutation_stdev); // Uninformed search variation
  allele_array.add(0,20/mutation_stdev); // Rate of informed search decay
  allele_array.add(0,20/mutation_stdev); // Rate of site fidelity
  allele_array.add(0,20/mutation_stdev); // Rate of laying pheremone
  allele_array.add(0,20/mutation_stdev); // Rate of pheremone decay

  // Create the template genome using the phenotype map we just made.
  GARealGenome genome(allele_array, objective);
  genome.crossover(GARealUniformCrossover);
  genome.mutator(GARealGaussianMutator);

  // Now create the GA using the genome and run it. We'll use sigma truncation
  // scaling so that we can handle negative objective scores.
  GASimpleGA ga(genome);
  GALinearScaling scaling;
  ga.maximize();		// Maximize the objective
  ga.populationSize(population_size);
  ga.nGenerations(n_generations);
  ga.pMutation(mutation_rate);
  ga.pCrossover(crossover_rate);
  ga.scaling(scaling);
  ga.elitist(gaFalse);
  if(mpi_rank == 0)
    ga.scoreFilename("evolution.txt");
  else
    ga.scoreFilename("/dev/null");
  ga.recordDiversity(gaTrue);
  ga.scoreFrequency(1);
  ga.flushFrequency(1);
  ga.selectScores(GAStatistics::AllScores);

  // Pass MPI data to the GA class
  ga.mpi_rank(mpi_rank);
  ga.mpi_tasks(mpi_tasks);
  //ga.evolve(seed);

    // Name the results file with the current time and date
 time_t t = time(0);   // get time now
    struct tm * now = localtime( & t );
    stringstream ss;

    ss << "results/CPFA-evolution-"
       << "powerlaw-6rovers" << '-'
       <<GIT_BRANCH<<"-"<<GIT_COMMIT_HASH<<"-"
       << (now->tm_year) << '-'
       << (now->tm_mon + 1) << '-'
       <<  now->tm_mday << '-'
       <<  now->tm_hour << '-'
       <<  now->tm_min << '-'
       <<  now->tm_sec << ".csv";

    string results_file_name = ss.str();

    if (mpi_rank == 0)
      {
    // Write output file header
    ofstream results_output_stream;
	results_output_stream.open(results_file_name, ios::app);
	results_output_stream << "Generation"
			      << ", " << "Convergence"
			      << ", " << "Mean"
			      << ", " << "Maximum"
			      << ", " << "Minimum"
			      << ", " << "Standard Deviation"
			      << ", " << "Diversity"
			      << ", " << "ProbabilityOfSwitchingToSearching"
			      << ", " << "ProbabilityOfReturningToNest"
			      << ", " << "UninformedSearchVariation"
			      << ", " << "RateOfInformedSearchDecay"
			      << ", " << "RateOfSiteFidelity"
			      << ", " << "RateOfLayingPheromone"
			      << ", " << "RateOfPheromoneDecay";

	results_output_stream << endl;
	results_output_stream.close();
      }

	// initialize the ga since we are not using the evolve function
	ga.initialize();

    while(!ga.done()){
    ga.step();

    if(mpi_rank == 0)
      {
	printf("Generation %d.\n", ga.generation());
	ofstream results_output_stream;
	results_output_stream.open(results_file_name, ios::app);
       results_output_stream << ga.statistics().generation()
			      << ", " << ga.statistics().convergence()
			      << ", " << ga.statistics().current(GAStatistics::Mean)
			      << ", " << ga.statistics().current(GAStatistics::Maximum)
			      << ", " << ga.statistics().current(GAStatistics::Minimum)
			      << ", " << ga.statistics().current(GAStatistics::Deviation)
			      << ", " << ga.statistics().current(GAStatistics::Diversity);

	  for (int i = 0; i < GENOME_SIZE; i++)
	    results_output_stream << ", " << dynamic_cast<const GARealGenome&>(ga.population().best()).gene(i)*mutation_stdev;

	results_output_stream << endl;
	results_output_stream.close();
      }
  }

  // Display the GA's progress
  if(mpi_rank == 0)
    {
      cout << ga.statistics() << " " << ga.parameters() << endl;
    }

  MPI_Finalize();

  end = std::chrono::system_clock::now();

  std::chrono::duration<double> elapsed_seconds = end-start;

  if(mpi_rank == 0)
    printf("Run time was %f seconds\n", elapsed_seconds.count());

  return 0;
  }

// The objective function used by GAlib

float objective(GAGenome &c)
{
  float avg = 0;

  // For timing
  std::chrono::time_point<std::chrono::system_clock> start, end;
  start = std::chrono::system_clock::now();

  for (int i = 0; i < n_trials; i++) avg += LaunchARGoS(c);

  avg /= n_trials;
  end = std::chrono::system_clock::now();

  std::chrono::duration<double> elapsed_seconds = end-start;

  MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);

      char hostname[1024];
      hostname[1023] = '\0';
      gethostname(hostname, 1023);

      printf("Worker %d on %s evaluated genome [", mpi_rank, hostname);
      for (int i = 0; i < GENOME_SIZE; i++)
	printf("%f ",dynamic_cast<const GARealGenome&>(c).gene(i)*mutation_stdev);
      printf("] in %f seconds. ", elapsed_seconds.count());
      printf("Fitness: %f.\n", avg );
      return avg;
}

/*
 * Launch ARGoS to evaluate a genome.
 */
float LaunchARGoS(GAGenome& c_genome)
{
  float fitness = 0;

  int    ShmID;
  float*   ShmPTR;

  // Allocate the shared memory to use between us and the child argos process
  ShmID = shmget(IPC_PRIVATE, sizeof(float), IPC_CREAT | 0666);
  if (ShmID < 0) {
    printf("*** shmget error (server) ***\n");
  }

  ShmPTR = (float*) shmat(ShmID, NULL, 0);
  if ((float*) ShmPTR == (float*)-1) {
    printf("*** shmat error (server) ***\n");
  }

  *ShmPTR = 0;

  pid_t pid = fork();

  if (pid == 0)
    {
      // In child process - run the argos3 simulation

      /* Convert the received genome to the actual genome type */
      GARealGenome& cRealGenome = dynamic_cast<GARealGenome&>(c_genome);

      Real* cpfa_genome = new Real[GENOME_SIZE];

      for (int i = 0; i < GENOME_SIZE; i++)
	cpfa_genome[i] = cRealGenome.gene(i)*mutation_stdev; // Convert back from genes transormed to simulate mutation stdev

      /*
      printf("%s: worker %d started a genome evaluation. (%f, %f, %f, %f, %f, %f, %f)\n", hostname, mpi_rank,
	     cpfa_genome[0],
	     cpfa_genome[1],
	     cpfa_genome[2],
	     cpfa_genome[3],
	     cpfa_genome[4],
	     cpfa_genome[5],
	     cpfa_genome[6]);
      */

      /* Redirect LOG and LOGERR to dedicated files to prevent clutter on the screen */

      std::ofstream cLOGFile("argos_logs/ARGoS_LOG_" + ToString(::getpid()), std::ios::out);
      LOG.DisableColoredOutput();
      LOG.GetStream().rdbuf(cLOGFile.rdbuf());
      std::ofstream cLOGERRFile("argos_logs/ARGoS_LOGERR_" + ToString(::getpid()), std::ios::out);
      LOGERR.DisableColoredOutput();
      LOGERR.GetStream().rdbuf(cLOGERRFile.rdbuf());

      /*
       * Initialize ARGoS
       */
      /* The CSimulator class of ARGoS is a singleton. Therefore, to
       * manipulate an ARGoS experiment, it is enough to get its instance */
      argos::CSimulator& cSimulator = argos::CSimulator::GetInstance();

      /* Set the .argos configuration file
       * This is a relative path which assumed that you launch the executable
       * from argos3-examples (as said also in the README) */
      cSimulator.SetExperimentFileName("experiments/CPFA.xml");
      /* Load it to configure ARGoS */

      cSimulator.LoadExperiment();

      /* Get a reference to the loop functions */
      CPFA_loop_functions& cLoopFunctions = dynamic_cast<CPFA_loop_functions&>(cSimulator.GetLoopFunctions());

      /* This internally calls also CEvolutionLoopFunctions::Reset(). */
      // cSimulator.Reset();

      /* Configure the controller with the genome */
      cLoopFunctions.ConfigureFromGenome(cpfa_genome);

      /* Run the experiment */
      cSimulator.Execute();

      /* Update performance */

      *ShmPTR = cLoopFunctions.Score();;

      //cLoopFunctions.Destroy();
      cSimulator.Destroy();

	delete [] cpfa_genome;

      _Exit(0);
    }

  // In parent - wait for child to finish
  int status = wait(&status);

  fitness = *ShmPTR;

  // Release shared memory
  shmdt((void *) ShmPTR);
  shmctl(ShmID, IPC_RMID, NULL);

  /* Return the result of the evaluation */
  return fitness;
}
<commit_msg>Restrict the waypoint gene to [0,2]<commit_after>#include <stdio.h>
#include <math.h>
#include <cstdlib>
#include <unistd.h> // For usleep

// For shared memory management
#include <sys/mman.h>

#include <ctime> // For clock()
#include <chrono> // For clock()
#include <mpi.h>

#include <sys/wait.h> // For wait(pid)

#include <limits> // For float max

// GA MPI Headers
#include <ga-mpi/ga.h>
#include <ga-mpi/std_stream.h>
#include <ga-mpi/GARealGenome.h>
#include <ga-mpi/GARealGenome.C>

// Argos headers
#include <argos3/core/simulator/simulator.h>
#include <argos3/core/simulator/loop_functions.h>
#include <source/CPFA/CPFA_loop_functions.h>

// For timing
#include <ctime> // For clock()
#include <chrono> // For clock()

// For shared variable between child and parent process
#include  <sys/ipc.h>
#include  <sys/shm.h>

float objective(GAGenome &);
float LaunchARGoS(GAGenome &);

int mpi_tasks, mpi_rank;

int n_trials = 10; // used by the objective function
double mutation_stdev = 0.1; // Used by LaunchArgos function.

int main(int argc, char **argv)
{
  std::chrono::time_point<std::chrono::system_clock> start, end;
  start = std::chrono::system_clock::now();

  srand(time(NULL));

  double mutation_rate = 0.1;
  double crossover_rate = 0.05;
  int population_size = 50;
  int n_generations = 100;

  char c='h';
  // Handle command line arguments
  while ((c = getopt (argc, argv, "tgp:")) != -1)
    switch (c)
      {
      case 't':
	n_trials = atoi(optarg);
	break;
      case 'g':
        n_generations = atoi(optarg);
        break;
      case 'p':
        population_size = atoi(optarg);
        break;
      case 'c':
	crossover_rate = strtod(optarg, NULL);
	break;
      case 'm':
        mutation_rate = strtod(optarg, NULL);
	break;
      case 'h':
	printf("Usage: %s -p {population size} -g {number of generations} -t {number of trials} -c {crossover rate} -m {mutation rate} -s {mutation standard deviation}", argv[0]);
	break;
      case '?':
        if (optopt == 'p')
          fprintf (stderr, "Option -%c requires an argument specifying the population size.\n", optopt);
	else if (optopt == 'g')
	  fprintf (stderr, "Option -%c requires an argument specifying the number of generations.\n", optopt);
	else if (optopt == 't')
	  fprintf (stderr, "Option -%c requires an argument specifying the number of trials.\n", optopt);
	else if (optopt == 'c')
	  fprintf (stderr, "Option -%c requires an argument specifying the crossover rate.\n", optopt);
	else if (optopt == 'm')
	  fprintf (stderr, "Option -%c requires an argument specifying the mutation rate.\n", optopt);
	else if (optopt == 's')
	  fprintf (stderr, "Option -%c requires an argument specifying the mutation standard deviation.\n", optopt);
        else if (isprint (optopt))
          fprintf (stderr, "Unknown option `-%c'.\n", optopt);
        else
          fprintf (stderr,
                   "Unknown option character `\\x%x'.\n",
                   optopt);
        return 1;
      default:
	printf("Usage: %s -p {population size} -g {number of generations} -t {number of trials} -c {crossover rate} -m {mutation rate} -s {mutation standard deviation}", argv[0]);
        abort ();
      }

  float max_float = std::numeric_limits<float>::max();

  // MPI init
  MPI_Init(&argc, &argv);
  MPI_Comm_size(MPI_COMM_WORLD, &mpi_tasks);
  MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);

  char hostname[1024];
  hostname[1023] = '\0';
  gethostname(hostname, 1023);
  printf("%s:\tworker %d ready.\n", hostname, mpi_rank);

  // See if we've been given a seed to use (for testing purposes).  When you
  // specify a random seed, the evolution will be exactly the same each time
  // you use that seed number
  unsigned int seed = 0;
  for(int i=1 ; i<argc ; i++)
    if(strcmp(argv[i++],"seed") == 0)
      seed = atoi(argv[i]);

  // popsize / mpi_tasks must be an integer
  population_size = mpi_tasks * int((double)population_size/(double)mpi_tasks+0.999);

  // Define the genome
  GARealAlleleSetArray allele_array;
  allele_array.add(0,1/mutation_stdev); // Probability of switching to search
  allele_array.add(0,1/mutation_stdev); // Probability of returning to nest
  allele_array.add(0,4*M_PI/mutation_stdev); // Uninformed search variation
  allele_array.add(0,20/mutation_stdev); // Rate of informed search decay
  allele_array.add(0,20/mutation_stdev); // Rate of site fidelity
  allele_array.add(0,20/mutation_stdev); // Rate of laying pheremone
  allele_array.add(0,2/mutation_stdev); // Rate of pheremone decay

  // Create the template genome using the phenotype map we just made.
  GARealGenome genome(allele_array, objective);
  genome.crossover(GARealUniformCrossover);
  genome.mutator(GARealGaussianMutator);

  // Now create the GA using the genome and run it. We'll use sigma truncation
  // scaling so that we can handle negative objective scores.
  GASimpleGA ga(genome);
  GALinearScaling scaling;
  ga.maximize();		// Maximize the objective
  ga.populationSize(population_size);
  ga.nGenerations(n_generations);
  ga.pMutation(mutation_rate);
  ga.pCrossover(crossover_rate);
  ga.scaling(scaling);
  ga.elitist(gaFalse);
  if(mpi_rank == 0)
    ga.scoreFilename("evolution.txt");
  else
    ga.scoreFilename("/dev/null");
  ga.recordDiversity(gaTrue);
  ga.scoreFrequency(1);
  ga.flushFrequency(1);
  ga.selectScores(GAStatistics::AllScores);

  // Pass MPI data to the GA class
  ga.mpi_rank(mpi_rank);
  ga.mpi_tasks(mpi_tasks);
  //ga.evolve(seed);

    // Name the results file with the current time and date
 time_t t = time(0);   // get time now
    struct tm * now = localtime( & t );
    stringstream ss;

    ss << "results/CPFA-evolution-"
       << "powerlaw-6rovers" << '-'
       <<GIT_BRANCH<<"-"<<GIT_COMMIT_HASH<<"-"
       << (now->tm_year) << '-'
       << (now->tm_mon + 1) << '-'
       <<  now->tm_mday << '-'
       <<  now->tm_hour << '-'
       <<  now->tm_min << '-'
       <<  now->tm_sec << ".csv";

    string results_file_name = ss.str();

    if (mpi_rank == 0)
      {
    // Write output file header
    ofstream results_output_stream;
	results_output_stream.open(results_file_name, ios::app);
	results_output_stream << "Generation"
			      << ", " << "Convergence"
			      << ", " << "Mean"
			      << ", " << "Maximum"
			      << ", " << "Minimum"
			      << ", " << "Standard Deviation"
			      << ", " << "Diversity"
			      << ", " << "ProbabilityOfSwitchingToSearching"
			      << ", " << "ProbabilityOfReturningToNest"
			      << ", " << "UninformedSearchVariation"
			      << ", " << "RateOfInformedSearchDecay"
			      << ", " << "RateOfSiteFidelity"
			      << ", " << "RateOfLayingPheromone"
			      << ", " << "RateOfPheromoneDecay";

	results_output_stream << endl;
	results_output_stream.close();
      }

	// initialize the ga since we are not using the evolve function
	ga.initialize();

    while(!ga.done()){
    ga.step();

    if(mpi_rank == 0)
      {
	printf("Generation %d.\n", ga.generation());
	ofstream results_output_stream;
	results_output_stream.open(results_file_name, ios::app);
       results_output_stream << ga.statistics().generation()
			      << ", " << ga.statistics().convergence()
			      << ", " << ga.statistics().current(GAStatistics::Mean)
			      << ", " << ga.statistics().current(GAStatistics::Maximum)
			      << ", " << ga.statistics().current(GAStatistics::Minimum)
			      << ", " << ga.statistics().current(GAStatistics::Deviation)
			      << ", " << ga.statistics().current(GAStatistics::Diversity);

	  for (int i = 0; i < GENOME_SIZE; i++)
	    results_output_stream << ", " << dynamic_cast<const GARealGenome&>(ga.population().best()).gene(i)*mutation_stdev;

	results_output_stream << endl;
	results_output_stream.close();
      }
  }

  // Display the GA's progress
  if(mpi_rank == 0)
    {
      cout << ga.statistics() << " " << ga.parameters() << endl;
    }

  MPI_Finalize();

  end = std::chrono::system_clock::now();

  std::chrono::duration<double> elapsed_seconds = end-start;

  if(mpi_rank == 0)
    printf("Run time was %f seconds\n", elapsed_seconds.count());

  return 0;
  }

// The objective function used by GAlib

float objective(GAGenome &c)
{
  float avg = 0;

  // For timing
  std::chrono::time_point<std::chrono::system_clock> start, end;
  start = std::chrono::system_clock::now();

  for (int i = 0; i < n_trials; i++) avg += LaunchARGoS(c);

  avg /= n_trials;
  end = std::chrono::system_clock::now();

  std::chrono::duration<double> elapsed_seconds = end-start;

  MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);

      char hostname[1024];
      hostname[1023] = '\0';
      gethostname(hostname, 1023);

      printf("Worker %d on %s evaluated genome [", mpi_rank, hostname);
      for (int i = 0; i < GENOME_SIZE; i++)
	printf("%f ",dynamic_cast<const GARealGenome&>(c).gene(i)*mutation_stdev);
      printf("] in %f seconds. ", elapsed_seconds.count());
      printf("Fitness: %f.\n", avg );
      return avg;
}

/*
 * Launch ARGoS to evaluate a genome.
 */
float LaunchARGoS(GAGenome& c_genome)
{
  float fitness = 0;

  int    ShmID;
  float*   ShmPTR;

  // Allocate the shared memory to use between us and the child argos process
  ShmID = shmget(IPC_PRIVATE, sizeof(float), IPC_CREAT | 0666);
  if (ShmID < 0) {
    printf("*** shmget error (server) ***\n");
  }

  ShmPTR = (float*) shmat(ShmID, NULL, 0);
  if ((float*) ShmPTR == (float*)-1) {
    printf("*** shmat error (server) ***\n");
  }

  *ShmPTR = 0;

  pid_t pid = fork();

  if (pid == 0)
    {
      // In child process - run the argos3 simulation

      /* Convert the received genome to the actual genome type */
      GARealGenome& cRealGenome = dynamic_cast<GARealGenome&>(c_genome);

      Real* cpfa_genome = new Real[GENOME_SIZE];

      for (int i = 0; i < GENOME_SIZE; i++)
	cpfa_genome[i] = cRealGenome.gene(i)*mutation_stdev; // Convert back from genes transormed to simulate mutation stdev

      /*
      printf("%s: worker %d started a genome evaluation. (%f, %f, %f, %f, %f, %f, %f)\n", hostname, mpi_rank,
	     cpfa_genome[0],
	     cpfa_genome[1],
	     cpfa_genome[2],
	     cpfa_genome[3],
	     cpfa_genome[4],
	     cpfa_genome[5],
	     cpfa_genome[6]);
      */

      /* Redirect LOG and LOGERR to dedicated files to prevent clutter on the screen */

      std::ofstream cLOGFile("argos_logs/ARGoS_LOG_" + ToString(::getpid()), std::ios::out);
      LOG.DisableColoredOutput();
      LOG.GetStream().rdbuf(cLOGFile.rdbuf());
      std::ofstream cLOGERRFile("argos_logs/ARGoS_LOGERR_" + ToString(::getpid()), std::ios::out);
      LOGERR.DisableColoredOutput();
      LOGERR.GetStream().rdbuf(cLOGERRFile.rdbuf());

      /*
       * Initialize ARGoS
       */
      /* The CSimulator class of ARGoS is a singleton. Therefore, to
       * manipulate an ARGoS experiment, it is enough to get its instance */
      argos::CSimulator& cSimulator = argos::CSimulator::GetInstance();

      /* Set the .argos configuration file
       * This is a relative path which assumed that you launch the executable
       * from argos3-examples (as said also in the README) */
      cSimulator.SetExperimentFileName("experiments/CPFA.xml");
      /* Load it to configure ARGoS */

      cSimulator.LoadExperiment();

      /* Get a reference to the loop functions */
      CPFA_loop_functions& cLoopFunctions = dynamic_cast<CPFA_loop_functions&>(cSimulator.GetLoopFunctions());

      /* This internally calls also CEvolutionLoopFunctions::Reset(). */
      // cSimulator.Reset();

      /* Configure the controller with the genome */
      cLoopFunctions.ConfigureFromGenome(cpfa_genome);

      /* Run the experiment */
      cSimulator.Execute();

      /* Update performance */

      *ShmPTR = cLoopFunctions.Score();;

      //cLoopFunctions.Destroy();
      cSimulator.Destroy();

	delete [] cpfa_genome;

      _Exit(0);
    }

  // In parent - wait for child to finish
  int status = wait(&status);

  fitness = *ShmPTR;

  // Release shared memory
  shmdt((void *) ShmPTR);
  shmctl(ShmID, IPC_RMID, NULL);

  /* Return the result of the evaluation */
  return fitness;
}
<|endoftext|>