<commit_before>/* ---------------------------------------------------------------------
 * Numenta Platform for Intelligent Computing (NuPIC)
 * Copyright (C) 2013, Numenta, Inc.  Unless you have an agreement
 * with Numenta, Inc., for a separate license for this software code, the
 * following terms and conditions apply:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero Public License version 3 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 * See the GNU Affero Public License for more details.
 *
 * You should have received a copy of the GNU Affero Public License
 * along with this program.  If not, see http://www.gnu.org/licenses.
 *
 * http://numenta.org/licenses/
 * ---------------------------------------------------------------------
 */

#include <algorithm> // sort
#include <iomanip>
#include <iostream>
#include <set>
#include <sstream>
#include <vector>

#include <assert.h>
#include <map>
#include <nupic/math/ArrayAlgo.hpp> // is_in
#include <nupic/math/StlIo.hpp>     // binary_save
#include <nupic/utils/Log.hpp>
#include <nupic/utils/Random.hpp>

#include <nupic/algorithms/Segment.hpp>

using namespace nupic::algorithms::Cells4;

//----------------------------------------------------------------------
/**
 * Utility routine. Given a src cell index, prints synapse as:
 *  [column, cell within col]
 */
void printSynapse(UInt srcCellIdx, UInt nCellsPerCol) {
  UInt col = (UInt)(srcCellIdx / nCellsPerCol);
  UInt cell = srcCellIdx - col * nCellsPerCol;
  std::cout << "[" << col << "," << cell << "]  ";
}

//----------------------------------------------------------------------
Segment::Segment(InSynapses _s, Real frequency, bool seqSegFlag,
                 Real permConnected, UInt iteration)
    : _totalActivations(1), _positiveActivations(1), _lastActiveIteration(0),
      _lastPosDutyCycle(1.0 / iteration), _lastPosDutyCycleIteration(iteration),
      _seqSegFlag(seqSegFlag), _frequency(frequency), _synapses(std::move(_s)),
      _nConnected(0) {
  for (UInt i = 0; i != _synapses.size(); ++i)
    if (_synapses[i].permanence() >= permConnected)
      ++_nConnected;

  std::sort(_synapses.begin(), _synapses.end(), InSynapseOrder());
  NTA_ASSERT(invariants());
}

//--------------------------------------------------------------------------------
Segment &Segment::operator=(const Segment &o) {
  if (&o != this) {
    _seqSegFlag = o._seqSegFlag;
    _frequency = o._frequency;
    _synapses = o._synapses;
    _nConnected = o._nConnected;
    _totalActivations = o._totalActivations;
    _positiveActivations = o._positiveActivations;
    _lastActiveIteration = o._lastActiveIteration;
    _lastPosDutyCycle = o._lastPosDutyCycle;
    _lastPosDutyCycleIteration = o._lastPosDutyCycleIteration;
  }
  NTA_ASSERT(invariants());
  return *this;
}

//--------------------------------------------------------------------------------
bool Segment::operator==(const Segment &other) const {
  if (_totalActivations != other._totalActivations ||
      _positiveActivations != other._positiveActivations ||
      _lastActiveIteration != other._lastActiveIteration ||
      !nearlyEqual(_lastPosDutyCycle, other._lastPosDutyCycle) ||
      _lastPosDutyCycleIteration  !=  other._lastPosDutyCycleIteration ||
      _seqSegFlag != other._seqSegFlag ||
      !nearlyEqual(_frequency, other._frequency) ||
      _nConnected != other._nConnected) {
    return false;
  }
  return _synapses == other._synapses;
}

//--------------------------------------------------------------------------------
Segment::Segment(const Segment &o)
    : _totalActivations(o._totalActivations),
      _positiveActivations(o._positiveActivations),
      _lastActiveIteration(o._lastActiveIteration),
      _lastPosDutyCycle(o._lastPosDutyCycle),
      _lastPosDutyCycleIteration(o._lastPosDutyCycleIteration),
      _seqSegFlag(o._seqSegFlag), _frequency(o._frequency),
      _synapses(o._synapses), _nConnected(o._nConnected) {
  NTA_ASSERT(invariants());
}

bool Segment::isActive(const CState &activities, Real permConnected,
                       UInt activationThreshold) const {
  { NTA_ASSERT(invariants()); }

  UInt activity = 0;

  if (_nConnected < activationThreshold)
    return false;

  // TODO: maintain nPermConnected incrementally??
  for (UInt i = 0; i != size() && activity < activationThreshold; ++i)
    if (_synapses[i].permanence() >= permConnected &&
        activities.isSet(_synapses[i].srcCellIdx()))
      activity++;

  return activity >= activationThreshold;
}

//----------------------------------------------------------------------
/**
 * Compute/update and return the positive activations duty cycle of
 * this segment. This is a measure of how often this segment is
 * providing good predictions.
 *
 */
Real Segment::dutyCycle(UInt iteration, bool active, bool readOnly) {
  { NTA_ASSERT(iteration > 0); }

  Real dutyCycle = 0.0;

  // For tier 0, compute it from total number of positive activations seen
  if (iteration <= _dutyCycleTiers[1]) {
    dutyCycle = ((Real)_positiveActivations) / iteration;
    if (!readOnly) {
      _lastPosDutyCycleIteration = iteration;
      _lastPosDutyCycle = dutyCycle;
    }
    return dutyCycle;
  }

  // How old is our update?
  UInt age = iteration - _lastPosDutyCycleIteration;

  // If it's already up to date we can return our cached value
  if (age == 0 && !active)
    return _lastPosDutyCycle;

  // Figure out which alpha we're using
  Real alpha = 0;
  for (UInt tierIdx = _numTiers - 1; tierIdx > 0; tierIdx--) {
    if (iteration > _dutyCycleTiers[tierIdx]) {
      alpha = _dutyCycleAlphas[tierIdx];
      break;
    }
  }

  // Update duty cycle
  dutyCycle = (Real)pow((Real64)(1.0 - alpha), (Real64)age) * _lastPosDutyCycle;
  if (active)
    dutyCycle += alpha;

  // Update the time we computed it
  if (!readOnly) {
    _lastPosDutyCycle = dutyCycle;
    _lastPosDutyCycleIteration = iteration;
  }

  return dutyCycle;
}

UInt Segment::computeActivity(const CState &activities, Real permConnected,
                              bool connectedSynapsesOnly) const

{
  { NTA_ASSERT(invariants()); }

  UInt activity = 0;

  if (connectedSynapsesOnly) {
    for (UInt i = 0; i != size(); ++i)
      if (activities.isSet(_synapses[i].srcCellIdx()) &&
          (_synapses[i].permanence() >= permConnected))
        activity++;
  } else {
    for (UInt i = 0; i != size(); ++i)
      if (activities.isSet(_synapses[i].srcCellIdx()))
        activity++;
  }

  return activity;
}

void Segment::addSynapses(const std::set<UInt> &srcCells, Real initStrength,
                          Real permConnected) {
  auto srcCellIdx = srcCells.begin();

  for (; srcCellIdx != srcCells.end(); ++srcCellIdx) {
    _synapses.push_back(InSynapse(*srcCellIdx, initStrength));
    if (initStrength >= permConnected)
      ++_nConnected;
  }

  sort(_synapses, InSynapseOrder());
  NTA_ASSERT(invariants()); // will catch non-unique synapses
}

void Segment::decaySynapses(Real decay, std::vector<UInt> &removed,
                            Real permConnected, bool doDecay) {
  NTA_ASSERT(invariants());

  if (_synapses.empty())
    return;

  static std::vector<UInt> del;
  del.clear(); // purge residual data

  for (UInt i = 0; i != _synapses.size(); ++i) {

    int wasConnected = (int)(_synapses[i].permanence() >= permConnected);

    if (_synapses[i].permanence() < decay) {

      removed.push_back(_synapses[i].srcCellIdx());
      del.push_back(i);

    } else if (doDecay) {
      _synapses[i].permanence() -= decay;
    }

    int isConnected = (int)(_synapses[i].permanence() >= permConnected);

    _nConnected += isConnected - wasConnected;
  }

  _removeSynapses(del);

  NTA_ASSERT(invariants());
}

//--------------------------------------------------------------------------------
/**
 * Subtract decay from each synapses' permanence value.
 * Synapses whose permanence drops below 0 are removed and their indices
 * are inserted into the "removed" list.
 *
 */
void Segment::decaySynapses2(Real decay, std::vector<UInt> &removed,
                             Real permConnected) {
  NTA_ASSERT(invariants());

  if (_synapses.empty())
    return;

  static std::vector<UInt> del;
  del.clear(); // purge residual data

  for (UInt i = 0; i != _synapses.size(); ++i) {

    // Remove synapse whose permanence will go to zero or below.
    if (_synapses[i].permanence() <= decay) {

      // If it was connected, reduce our connected count
      if (_synapses[i].permanence() >= permConnected)
        _nConnected--;

      // Add this synapse to list of synapses to be removed
      removed.push_back(_synapses[i].srcCellIdx());
      del.push_back(i);

    } else {

      _synapses[i].permanence() -= decay;

      // If it was connected and is now below permanence, reduce connected count
      if ((_synapses[i].permanence() + decay >= permConnected) &&
          (_synapses[i].permanence() < permConnected))
        _nConnected--;
    }
  }

  _removeSynapses(del);

  NTA_ASSERT(invariants());
}

//-----------------------------------------------------------------------
/**
 * Sort order for InSynapse's. Cells are sorted in order of increasing
 * permanence.
 *
 */
struct InPermanenceOrder {
  inline bool operator()(const InSynapse &a, const InSynapse &b) const {
    return a.permanence() < b.permanence();
  }
};

//-----------------------------------------------------------------------
/**
 * Sort order for list of source cell indices. Cells are sorted in order of
 * increasing source cell index.
 *
 */
struct InSrcCellOrder {
  inline bool operator()(const UInt a, const UInt b) const { return a < b; }
};

//----------------------------------------------------------------------
/**
 * Free up some synapses in this segment. We always free up inactive
 * synapses (lowest permanence freed up first) before we start to free
 * up active ones.
 */
void Segment::freeNSynapses(UInt numToFree,
                            std::vector<UInt> &inactiveSynapseIndices,
                            std::vector<UInt> &inactiveSegmentIndices,
                            std::vector<UInt> &activeSynapseIndices,
                            std::vector<UInt> &activeSegmentIndices,
                            std::vector<UInt> &removed, UInt verbosity,
                            UInt nCellsPerCol, Real permMax) {
  NTA_ASSERT(inactiveSegmentIndices.size() == inactiveSynapseIndices.size());
  NTA_ASSERT(activeSegmentIndices.size() == activeSynapseIndices.size());
  NTA_ASSERT(numToFree <= _synapses.size());
  NTA_ASSERT(numToFree <=
             (inactiveSegmentIndices.size() + activeSegmentIndices.size()));

  if (verbosity >= 4) {
    std::cout << "\nIn CPP freeNSynapses with numToFree = " << numToFree
              << ", inactiveSynapses = ";
    for (auto &inactiveSynapseIndice : inactiveSynapseIndices) {
      printSynapse(inactiveSynapseIndice, nCellsPerCol);
    }
    std::cout << "\n";
  }

  //----------------------------------------------------------------------
  // Collect candidate synapses for deletion

  // We first choose from inactive synapses, in order of increasing permanence
  InSynapses candidates;
  for (UInt i = 0; i < inactiveSegmentIndices.size(); i++) {
    // Put in *segment indices*, not source cell indices
    candidates.push_back(
        InSynapse(inactiveSegmentIndices[i],
                  _synapses[inactiveSegmentIndices[i]].permanence()));
  }

  // If we need more, choose from active synapses in order of increasing
  // permanence values. This set has lower priority than inactive synapses
  // so we add a constant permanence value for sorting purposes
  if (candidates.size() < numToFree) {
    for (UInt i = 0; i < activeSegmentIndices.size(); i++) {
      // Put in *segment indices*, not source cell indices
      candidates.push_back(
          InSynapse(activeSegmentIndices[i],
                    _synapses[activeSegmentIndices[i]].permanence() + permMax));
    }
  }

  // Now sort the list of candidate synapses
  std::stable_sort(candidates.begin(), candidates.end(), InPermanenceOrder());

  //----------------------------------------------------------------------
  // Create the final list of synapses we will remove
  static std::vector<UInt> del;
  del.clear(); // purge residual data
  for (UInt i = 0; i < numToFree; i++) {
    del.push_back(candidates[i].srcCellIdx());
    UInt cellIdx = _synapses[candidates[i].srcCellIdx()].srcCellIdx();
    removed.push_back(cellIdx);
  }

  // Debug statements
  if (verbosity >= 4) {
    std::cout << "Removing these synapses: ";
    for (auto &elem : removed) {
      printSynapse(elem, nCellsPerCol);
    }
    std::cout << "\n";

    std::cout << "Segment BEFORE remove synapses: ";
    print(std::cout, nCellsPerCol);
    std::cout << "\n";
  }

  //----------------------------------------------------------------------
  // Remove the synapses
  if (numToFree > 0) {
    std::sort(del.begin(), del.end(), InSrcCellOrder());
    _removeSynapses(del);
  }

  // Debug statements
  if (verbosity >= 4) {
    std::cout << "Segment AFTER remove synapses: ";
    print(std::cout, nCellsPerCol);
    std::cout << "\n";
  }
}

void Segment::print(std::ostream &outStream, UInt nCellsPerCol) const {
  outStream << (_seqSegFlag ? "True " : "False ") << "dc"
            << std::setprecision(4) << _lastPosDutyCycle << " ("
            << _positiveActivations << "/" << _totalActivations << ") ";
  for (UInt i = 0; i != _synapses.size(); ++i) {
    if (nCellsPerCol > 0) {
      UInt cellIdx = _synapses[i].srcCellIdx();
      UInt col = (UInt)(cellIdx / nCellsPerCol);
      UInt cell = cellIdx - col * nCellsPerCol;
      outStream << "[" << col << "," << cell << "]" << std::setprecision(4)
                << _synapses[i].permanence() << " ";
    } else {
      outStream << _synapses[i];
    }
    if (i < _synapses.size() -1)
      outStream << " ";
  }
  outStream << std::endl;
}

namespace nupic{
  namespace algorithms {
    namespace Cells4 {

std::ostream &operator<<(std::ostream &outStream, const Segment &seg) {
  seg.print(outStream);
  return outStream;
}

std::ostream &operator<<(std::ostream &outStream, const CState &cstate) {
  cstate.print(outStream);
  return outStream;
}

std::ostream &operator<<(std::ostream &outStream, const CStateIndexed &cstate) {
  cstate.print(outStream);
  return outStream;
}
} // namespace Cells4
} // namespace algorithms
} // namespace nupic
<commit_msg>Segment: fix equals redefinition<commit_after>/* ---------------------------------------------------------------------
 * Numenta Platform for Intelligent Computing (NuPIC)
 * Copyright (C) 2013, Numenta, Inc.  Unless you have an agreement
 * with Numenta, Inc., for a separate license for this software code, the
 * following terms and conditions apply:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero Public License version 3 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 * See the GNU Affero Public License for more details.
 *
 * You should have received a copy of the GNU Affero Public License
 * along with this program.  If not, see http://www.gnu.org/licenses.
 *
 * http://numenta.org/licenses/
 * ---------------------------------------------------------------------
 */

#include <algorithm> // sort
#include <iomanip>
#include <iostream>
#include <set>
#include <sstream>
#include <vector>

#include <assert.h>
#include <map>
#include <nupic/math/ArrayAlgo.hpp> // is_in
#include <nupic/math/StlIo.hpp>     // binary_save
#include <nupic/utils/Log.hpp>
#include <nupic/utils/Random.hpp>

#include <nupic/algorithms/Segment.hpp>

using namespace nupic::algorithms::Cells4;

//----------------------------------------------------------------------
/**
 * Utility routine. Given a src cell index, prints synapse as:
 *  [column, cell within col]
 */
void printSynapse(UInt srcCellIdx, UInt nCellsPerCol) {
  UInt col = (UInt)(srcCellIdx / nCellsPerCol);
  UInt cell = srcCellIdx - col * nCellsPerCol;
  std::cout << "[" << col << "," << cell << "]  ";
}

//----------------------------------------------------------------------
Segment::Segment(InSynapses _s, Real frequency, bool seqSegFlag,
                 Real permConnected, UInt iteration)
    : _totalActivations(1), _positiveActivations(1), _lastActiveIteration(0),
      _lastPosDutyCycle(1.0 / iteration), _lastPosDutyCycleIteration(iteration),
      _seqSegFlag(seqSegFlag), _frequency(frequency), _synapses(std::move(_s)),
      _nConnected(0) {
  for (UInt i = 0; i != _synapses.size(); ++i)
    if (_synapses[i].permanence() >= permConnected)
      ++_nConnected;

  std::sort(_synapses.begin(), _synapses.end(), InSynapseOrder());
  NTA_ASSERT(invariants());
}

//--------------------------------------------------------------------------------
Segment &Segment::operator=(const Segment &o) {
  if (&o != this) {
    _seqSegFlag = o._seqSegFlag;
    _frequency = o._frequency;
    _synapses = o._synapses;
    _nConnected = o._nConnected;
    _totalActivations = o._totalActivations;
    _positiveActivations = o._positiveActivations;
    _lastActiveIteration = o._lastActiveIteration;
    _lastPosDutyCycle = o._lastPosDutyCycle;
    _lastPosDutyCycleIteration = o._lastPosDutyCycleIteration;
  }
  NTA_ASSERT(invariants());
  return *this;
}

//--------------------------------------------------------------------------------
bool Segment::equals(const Segment &other) const {
  if (_totalActivations != other._totalActivations ||
      _positiveActivations != other._positiveActivations ||
      _lastActiveIteration != other._lastActiveIteration ||
      !nearlyEqual(_lastPosDutyCycle, other._lastPosDutyCycle) ||
      _lastPosDutyCycleIteration  !=  other._lastPosDutyCycleIteration ||
      _seqSegFlag != other._seqSegFlag ||
      !nearlyEqual(_frequency, other._frequency) ||
      _nConnected != other._nConnected) {
    return false;
  }
  return _synapses == other._synapses;
}

//--------------------------------------------------------------------------------
Segment::Segment(const Segment &o)
    : _totalActivations(o._totalActivations),
      _positiveActivations(o._positiveActivations),
      _lastActiveIteration(o._lastActiveIteration),
      _lastPosDutyCycle(o._lastPosDutyCycle),
      _lastPosDutyCycleIteration(o._lastPosDutyCycleIteration),
      _seqSegFlag(o._seqSegFlag), _frequency(o._frequency),
      _synapses(o._synapses), _nConnected(o._nConnected) {
  NTA_ASSERT(invariants());
}

bool Segment::isActive(const CState &activities, Real permConnected,
                       UInt activationThreshold) const {
  { NTA_ASSERT(invariants()); }

  UInt activity = 0;

  if (_nConnected < activationThreshold)
    return false;

  // TODO: maintain nPermConnected incrementally??
  for (UInt i = 0; i != size() && activity < activationThreshold; ++i)
    if (_synapses[i].permanence() >= permConnected &&
        activities.isSet(_synapses[i].srcCellIdx()))
      activity++;

  return activity >= activationThreshold;
}

//----------------------------------------------------------------------
/**
 * Compute/update and return the positive activations duty cycle of
 * this segment. This is a measure of how often this segment is
 * providing good predictions.
 *
 */
Real Segment::dutyCycle(UInt iteration, bool active, bool readOnly) {
  { NTA_ASSERT(iteration > 0); }

  Real dutyCycle = 0.0;

  // For tier 0, compute it from total number of positive activations seen
  if (iteration <= _dutyCycleTiers[1]) {
    dutyCycle = ((Real)_positiveActivations) / iteration;
    if (!readOnly) {
      _lastPosDutyCycleIteration = iteration;
      _lastPosDutyCycle = dutyCycle;
    }
    return dutyCycle;
  }

  // How old is our update?
  UInt age = iteration - _lastPosDutyCycleIteration;

  // If it's already up to date we can return our cached value
  if (age == 0 && !active)
    return _lastPosDutyCycle;

  // Figure out which alpha we're using
  Real alpha = 0;
  for (UInt tierIdx = _numTiers - 1; tierIdx > 0; tierIdx--) {
    if (iteration > _dutyCycleTiers[tierIdx]) {
      alpha = _dutyCycleAlphas[tierIdx];
      break;
    }
  }

  // Update duty cycle
  dutyCycle = (Real)pow((Real64)(1.0 - alpha), (Real64)age) * _lastPosDutyCycle;
  if (active)
    dutyCycle += alpha;

  // Update the time we computed it
  if (!readOnly) {
    _lastPosDutyCycle = dutyCycle;
    _lastPosDutyCycleIteration = iteration;
  }

  return dutyCycle;
}

UInt Segment::computeActivity(const CState &activities, Real permConnected,
                              bool connectedSynapsesOnly) const

{
  { NTA_ASSERT(invariants()); }

  UInt activity = 0;

  if (connectedSynapsesOnly) {
    for (UInt i = 0; i != size(); ++i)
      if (activities.isSet(_synapses[i].srcCellIdx()) &&
          (_synapses[i].permanence() >= permConnected))
        activity++;
  } else {
    for (UInt i = 0; i != size(); ++i)
      if (activities.isSet(_synapses[i].srcCellIdx()))
        activity++;
  }

  return activity;
}

void Segment::addSynapses(const std::set<UInt> &srcCells, Real initStrength,
                          Real permConnected) {
  auto srcCellIdx = srcCells.begin();

  for (; srcCellIdx != srcCells.end(); ++srcCellIdx) {
    _synapses.push_back(InSynapse(*srcCellIdx, initStrength));
    if (initStrength >= permConnected)
      ++_nConnected;
  }

  sort(_synapses, InSynapseOrder());
  NTA_ASSERT(invariants()); // will catch non-unique synapses
}

void Segment::decaySynapses(Real decay, std::vector<UInt> &removed,
                            Real permConnected, bool doDecay) {
  NTA_ASSERT(invariants());

  if (_synapses.empty())
    return;

  static std::vector<UInt> del;
  del.clear(); // purge residual data

  for (UInt i = 0; i != _synapses.size(); ++i) {

    int wasConnected = (int)(_synapses[i].permanence() >= permConnected);

    if (_synapses[i].permanence() < decay) {

      removed.push_back(_synapses[i].srcCellIdx());
      del.push_back(i);

    } else if (doDecay) {
      _synapses[i].permanence() -= decay;
    }

    int isConnected = (int)(_synapses[i].permanence() >= permConnected);

    _nConnected += isConnected - wasConnected;
  }

  _removeSynapses(del);

  NTA_ASSERT(invariants());
}

//--------------------------------------------------------------------------------
/**
 * Subtract decay from each synapses' permanence value.
 * Synapses whose permanence drops below 0 are removed and their indices
 * are inserted into the "removed" list.
 *
 */
void Segment::decaySynapses2(Real decay, std::vector<UInt> &removed,
                             Real permConnected) {
  NTA_ASSERT(invariants());

  if (_synapses.empty())
    return;

  static std::vector<UInt> del;
  del.clear(); // purge residual data

  for (UInt i = 0; i != _synapses.size(); ++i) {

    // Remove synapse whose permanence will go to zero or below.
    if (_synapses[i].permanence() <= decay) {

      // If it was connected, reduce our connected count
      if (_synapses[i].permanence() >= permConnected)
        _nConnected--;

      // Add this synapse to list of synapses to be removed
      removed.push_back(_synapses[i].srcCellIdx());
      del.push_back(i);

    } else {

      _synapses[i].permanence() -= decay;

      // If it was connected and is now below permanence, reduce connected count
      if ((_synapses[i].permanence() + decay >= permConnected) &&
          (_synapses[i].permanence() < permConnected))
        _nConnected--;
    }
  }

  _removeSynapses(del);

  NTA_ASSERT(invariants());
}

//-----------------------------------------------------------------------
/**
 * Sort order for InSynapse's. Cells are sorted in order of increasing
 * permanence.
 *
 */
struct InPermanenceOrder {
  inline bool operator()(const InSynapse &a, const InSynapse &b) const {
    return a.permanence() < b.permanence();
  }
};

//-----------------------------------------------------------------------
/**
 * Sort order for list of source cell indices. Cells are sorted in order of
 * increasing source cell index.
 *
 */
struct InSrcCellOrder {
  inline bool operator()(const UInt a, const UInt b) const { return a < b; }
};

//----------------------------------------------------------------------
/**
 * Free up some synapses in this segment. We always free up inactive
 * synapses (lowest permanence freed up first) before we start to free
 * up active ones.
 */
void Segment::freeNSynapses(UInt numToFree,
                            std::vector<UInt> &inactiveSynapseIndices,
                            std::vector<UInt> &inactiveSegmentIndices,
                            std::vector<UInt> &activeSynapseIndices,
                            std::vector<UInt> &activeSegmentIndices,
                            std::vector<UInt> &removed, UInt verbosity,
                            UInt nCellsPerCol, Real permMax) {
  NTA_ASSERT(inactiveSegmentIndices.size() == inactiveSynapseIndices.size());
  NTA_ASSERT(activeSegmentIndices.size() == activeSynapseIndices.size());
  NTA_ASSERT(numToFree <= _synapses.size());
  NTA_ASSERT(numToFree <=
             (inactiveSegmentIndices.size() + activeSegmentIndices.size()));

  if (verbosity >= 4) {
    std::cout << "\nIn CPP freeNSynapses with numToFree = " << numToFree
              << ", inactiveSynapses = ";
    for (auto &inactiveSynapseIndice : inactiveSynapseIndices) {
      printSynapse(inactiveSynapseIndice, nCellsPerCol);
    }
    std::cout << "\n";
  }

  //----------------------------------------------------------------------
  // Collect candidate synapses for deletion

  // We first choose from inactive synapses, in order of increasing permanence
  InSynapses candidates;
  for (UInt i = 0; i < inactiveSegmentIndices.size(); i++) {
    // Put in *segment indices*, not source cell indices
    candidates.push_back(
        InSynapse(inactiveSegmentIndices[i],
                  _synapses[inactiveSegmentIndices[i]].permanence()));
  }

  // If we need more, choose from active synapses in order of increasing
  // permanence values. This set has lower priority than inactive synapses
  // so we add a constant permanence value for sorting purposes
  if (candidates.size() < numToFree) {
    for (UInt i = 0; i < activeSegmentIndices.size(); i++) {
      // Put in *segment indices*, not source cell indices
      candidates.push_back(
          InSynapse(activeSegmentIndices[i],
                    _synapses[activeSegmentIndices[i]].permanence() + permMax));
    }
  }

  // Now sort the list of candidate synapses
  std::stable_sort(candidates.begin(), candidates.end(), InPermanenceOrder());

  //----------------------------------------------------------------------
  // Create the final list of synapses we will remove
  static std::vector<UInt> del;
  del.clear(); // purge residual data
  for (UInt i = 0; i < numToFree; i++) {
    del.push_back(candidates[i].srcCellIdx());
    UInt cellIdx = _synapses[candidates[i].srcCellIdx()].srcCellIdx();
    removed.push_back(cellIdx);
  }

  // Debug statements
  if (verbosity >= 4) {
    std::cout << "Removing these synapses: ";
    for (auto &elem : removed) {
      printSynapse(elem, nCellsPerCol);
    }
    std::cout << "\n";

    std::cout << "Segment BEFORE remove synapses: ";
    print(std::cout, nCellsPerCol);
    std::cout << "\n";
  }

  //----------------------------------------------------------------------
  // Remove the synapses
  if (numToFree > 0) {
    std::sort(del.begin(), del.end(), InSrcCellOrder());
    _removeSynapses(del);
  }

  // Debug statements
  if (verbosity >= 4) {
    std::cout << "Segment AFTER remove synapses: ";
    print(std::cout, nCellsPerCol);
    std::cout << "\n";
  }
}

void Segment::print(std::ostream &outStream, UInt nCellsPerCol) const {
  outStream << (_seqSegFlag ? "True " : "False ") << "dc"
            << std::setprecision(4) << _lastPosDutyCycle << " ("
            << _positiveActivations << "/" << _totalActivations << ") ";
  for (UInt i = 0; i != _synapses.size(); ++i) {
    if (nCellsPerCol > 0) {
      UInt cellIdx = _synapses[i].srcCellIdx();
      UInt col = (UInt)(cellIdx / nCellsPerCol);
      UInt cell = cellIdx - col * nCellsPerCol;
      outStream << "[" << col << "," << cell << "]" << std::setprecision(4)
                << _synapses[i].permanence() << " ";
    } else {
      outStream << _synapses[i];
    }
    if (i < _synapses.size() -1)
      outStream << " ";
  }
  outStream << std::endl;
}

namespace nupic{
  namespace algorithms {
    namespace Cells4 {

std::ostream &operator<<(std::ostream &outStream, const Segment &seg) {
  seg.print(outStream);
  return outStream;
}

std::ostream &operator<<(std::ostream &outStream, const CState &cstate) {
  cstate.print(outStream);
  return outStream;
}

std::ostream &operator<<(std::ostream &outStream, const CStateIndexed &cstate) {
  cstate.print(outStream);
  return outStream;
}
} // namespace Cells4
} // namespace algorithms
} // namespace nupic
<|endoftext|>


<commit_before>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: i_nnfinder.hxx,v $
 *
 *  $Revision: 1.3 $
 *
 *  last change: $Author: hr $ $Date: 2007-11-02 15:45:59 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#ifndef ARY_IDL_NNFINDER_HXX
#define ARY_IDL_NNFINDER_HXX

// USED SERVICES
#include "is_ce.hxx"

namespace ary
{
namespace idl
{

/** Gives context info for tree search functions.

    @collab ->ary::Search_SubTree<>()
    @collab ->ary::Search_SubTree_UpTillRoot<>()
*/
class Find_ModuleNode
{
  public:
    typedef Ce_id                           id_type;
    typedef StringVector::const_iterator    name_iterator;

    // LIFECYCLE
                        Find_ModuleNode(
                            const Ce_Storage &  i_rStorage,
                            name_iterator       it_begin,
                            name_iterator       it_end,
                            const String &      i_sName )
                            :   rStorage(i_rStorage),
                                itBegin(it_begin),
                                itEnd(it_end),
                                sName2Search(i_sName) { if (itBegin != itEnd ? (*itBegin).empty() : false) ++itBegin; }
    // OPERATIONS
    const Module *      operator()(
                            id_type             i_id ) const
                            { return i_id.IsValid()
                                        ?   & ary_cast<Module>(rStorage[i_id])
                                        :   0; }

    name_iterator       Begin() const           { return itBegin; }
    name_iterator       End() const             { return itEnd; }
    const String &      Name2Search() const     { return sName2Search; }

  private:
    // DATA
    const Ce_Storage &  rStorage;
    name_iterator       itBegin;
    name_iterator       itEnd;
    String              sName2Search;
};

class Types_forSetCe_Id
{
  public:
    typedef Ce_id                           element_type;
    typedef Ce_Storage                      find_type;

    //  KORR_FUTURE: Check, if this sorting is right or the ary standard
    //  sorting should be used.
    struct sort_type
    {
                        sort_type(
                            const find_type &   i_rFinder )
                                                : rFinder(i_rFinder) {}
        bool            operator()(
                            const element_type   &
                                                i_r1,
                            const element_type   &
                                                i_r2 ) const
        {
            return rFinder[i_r1].LocalName()
                   < rFinder[i_r2].LocalName();
        }

      private:
        const find_type &     rFinder;

    };
};

}   // namespace idl
}   // namespace ary
#endif
<commit_msg>INTEGRATION: CWS changefileheader (1.3.22); FILE MERGED 2008/03/28 16:01:45 rt 1.3.22.1: #i87441# Change license header to LPGL v3.<commit_after>/*************************************************************************
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * Copyright 2008 by Sun Microsystems, Inc.
 *
 * OpenOffice.org - a multi-platform office productivity suite
 *
 * $RCSfile: i_nnfinder.hxx,v $
 * $Revision: 1.4 $
 *
 * This file is part of OpenOffice.org.
 *
 * OpenOffice.org is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License version 3
 * only, as published by the Free Software Foundation.
 *
 * OpenOffice.org is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License version 3 for more details
 * (a copy is included in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU Lesser General Public License
 * version 3 along with OpenOffice.org.  If not, see
 * <http://www.openoffice.org/license.html>
 * for a copy of the LGPLv3 License.
 *
 ************************************************************************/

#ifndef ARY_IDL_NNFINDER_HXX
#define ARY_IDL_NNFINDER_HXX

// USED SERVICES
#include "is_ce.hxx"

namespace ary
{
namespace idl
{

/** Gives context info for tree search functions.

    @collab ->ary::Search_SubTree<>()
    @collab ->ary::Search_SubTree_UpTillRoot<>()
*/
class Find_ModuleNode
{
  public:
    typedef Ce_id                           id_type;
    typedef StringVector::const_iterator    name_iterator;

    // LIFECYCLE
                        Find_ModuleNode(
                            const Ce_Storage &  i_rStorage,
                            name_iterator       it_begin,
                            name_iterator       it_end,
                            const String &      i_sName )
                            :   rStorage(i_rStorage),
                                itBegin(it_begin),
                                itEnd(it_end),
                                sName2Search(i_sName) { if (itBegin != itEnd ? (*itBegin).empty() : false) ++itBegin; }
    // OPERATIONS
    const Module *      operator()(
                            id_type             i_id ) const
                            { return i_id.IsValid()
                                        ?   & ary_cast<Module>(rStorage[i_id])
                                        :   0; }

    name_iterator       Begin() const           { return itBegin; }
    name_iterator       End() const             { return itEnd; }
    const String &      Name2Search() const     { return sName2Search; }

  private:
    // DATA
    const Ce_Storage &  rStorage;
    name_iterator       itBegin;
    name_iterator       itEnd;
    String              sName2Search;
};

class Types_forSetCe_Id
{
  public:
    typedef Ce_id                           element_type;
    typedef Ce_Storage                      find_type;

    //  KORR_FUTURE: Check, if this sorting is right or the ary standard
    //  sorting should be used.
    struct sort_type
    {
                        sort_type(
                            const find_type &   i_rFinder )
                                                : rFinder(i_rFinder) {}
        bool            operator()(
                            const element_type   &
                                                i_r1,
                            const element_type   &
                                                i_r2 ) const
        {
            return rFinder[i_r1].LocalName()
                   < rFinder[i_r2].LocalName();
        }

      private:
        const find_type &     rFinder;

    };
};

}   // namespace idl
}   // namespace ary
#endif
<|endoftext|>


<commit_before>/*************************************************************************
 *
 *  $RCSfile: cross_refs.hxx,v $
 *
 *  $Revision: 1.1 $
 *
 *  last change: $Author: np $ $Date: 2002-11-01 17:13:40 $
 *
 *  The Contents of this file are made available subject to the terms of
 *  either of the following licenses
 *
 *         - GNU Lesser General Public License Version 2.1
 *         - Sun Industry Standards Source License Version 1.1
 *
 *  Sun Microsystems Inc., October, 2000
 *
 *  GNU Lesser General Public License Version 2.1
 *  =============================================
 *  Copyright 2000 by Sun Microsystems, Inc.
 *  901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License version 2.1, as published by the Free Software Foundation.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *  MA  02111-1307  USA
 *
 *
 *  Sun Industry Standards Source License Version 1.1
 *  =================================================
 *  The contents of this file are subject to the Sun Industry Standards
 *  Source License Version 1.1 (the "License"); You may not use this file
 *  except in compliance with the License. You may obtain a copy of the
 *  License at http://www.openoffice.org/license.html.
 *
 *  Software provided under this License is provided on an "AS IS" basis,
 *  WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
 *  WITHOUT LIMITATION, WARRANTIES THAT THE SOFTWARE IS FREE OF DEFECTS,
 *  MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE, OR NON-INFRINGING.
 *  See the License for the specific provisions governing your rights and
 *  obligations concerning the Software.
 *
 *  The Initial Developer of the Original Code is: Sun Microsystems, Inc.
 *
 *  Copyright: 2000 by Sun Microsystems, Inc.
 *
 *  All Rights Reserved.
 *
 *  Contributor(s): _______________________________________
 *
 *
 ************************************************************************/

#ifndef ARY_CROSS_REFS_HXX
#define ARY_CROSS_REFS_HXX

// USED SERVICES
    // BASE CLASSES
    // COMPONENTS
    // PARAMETERS
#include "sorted_idset.hxx"

template <class VALUE_LIST, class TYPES>
class CrossReferences
{
  public:
    typedef TYPES::element_type         element;

    /// Checks for double occurences
    void                Add(
                            VALUE_LIST::index_type
                                                i_nPosition
                            const element &     i_rElem );
    void                Get_List(
                            Dyn_StdConstIterator<element> &
                                                o_rResult ) const;
  private:
    SortedIdSet<TYPES>  aData[VALUE_LIST::max];
};

namespace ary
{

template <class TYPES>
class SortedIdSet
{
  public:
    typedef typename TYPES::element_type   element;
    typedef typename TYPES::sort_type      sorter;
    typedef typename TYPES::find_type      finder;

                        SortedIdSet(
                            const finder &      i_rFinder )
                                                : aSorter(i_rFinder),
                                                  aData(aSorter) {}
                        ~SortedIdSet()          {}

    void                Get_Begin(
                            Dyn_StdConstIterator<element> &
                                                o_rResult )
                                                { o_rResult = new SCI_Set<FINDER>(aData); }
    void                Add(
                            const element &     i_rElement )
                                                { aData.insert(i_rElement); }

  private:
    typedef std::set<element, sorter>       Set;

    // DATA
    sorter              aSorter;
    Set                 aData;
};

}   // namespace ary

#endif

<commit_msg>INTEGRATION: CWS ooo19126 (1.1.132); FILE MERGED 2005/09/05 13:10:13 rt 1.1.132.1: #i54170# Change license header: remove SISSL<commit_after>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: cross_refs.hxx,v $
 *
 *  $Revision: 1.2 $
 *
 *  last change: $Author: rt $ $Date: 2005-09-07 16:57:57 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#ifndef ARY_CROSS_REFS_HXX
#define ARY_CROSS_REFS_HXX

// USED SERVICES
    // BASE CLASSES
    // COMPONENTS
    // PARAMETERS
#include "sorted_idset.hxx"

template <class VALUE_LIST, class TYPES>
class CrossReferences
{
  public:
    typedef TYPES::element_type         element;

    /// Checks for double occurences
    void                Add(
                            VALUE_LIST::index_type
                                                i_nPosition
                            const element &     i_rElem );
    void                Get_List(
                            Dyn_StdConstIterator<element> &
                                                o_rResult ) const;
  private:
    SortedIdSet<TYPES>  aData[VALUE_LIST::max];
};

namespace ary
{

template <class TYPES>
class SortedIdSet
{
  public:
    typedef typename TYPES::element_type   element;
    typedef typename TYPES::sort_type      sorter;
    typedef typename TYPES::find_type      finder;

                        SortedIdSet(
                            const finder &      i_rFinder )
                                                : aSorter(i_rFinder),
                                                  aData(aSorter) {}
                        ~SortedIdSet()          {}

    void                Get_Begin(
                            Dyn_StdConstIterator<element> &
                                                o_rResult )
                                                { o_rResult = new SCI_Set<FINDER>(aData); }
    void                Add(
                            const element &     i_rElement )
                                                { aData.insert(i_rElement); }

  private:
    typedef std::set<element, sorter>       Set;

    // DATA
    sorter              aSorter;
    Set                 aData;
};

}   // namespace ary

#endif

<|endoftext|>


<commit_before>#include "PageBuilder.h"

PageBuilder::PageBuilder(const std::set<PageInfo> &Pages)
{
    pages = Pages;
}

int PageBuilder::build(const PageInfo &PageToBuild)
{
    pageToBuild = PageToBuild;

    std::cout << std::endl;

    //ensures content and template files exist
    if(!std::ifstream(pageToBuild.contentPath.str()))
    {
        std::cout << "error: cannot build " << pageToBuild.pagePath << " as content file " << pageToBuild.contentPath << " does not exist" << std::endl;
        return 1;
    }
    if(!std::ifstream(pageToBuild.templatePath.str()))
    {
        std::cout << "error: cannot build " << pageToBuild.pagePath << " as template file " << pageToBuild.templatePath << " does not exist." << std::endl;
        return 1;
    }

    std::cout << "building page " << pageToBuild.pagePath << std::endl;

    //makes sure variables are at default values
    codeBlockDepth = htmlCommentDepth = 0;
    indentAmount = "";
    contentAdded = 0;
    processedPage.clear();
    processedPage.str(std::string());
    pageDeps.clear();
    contentAdded = 0;

    //adds content and template paths to dependencies
    pageDeps.insert(pageToBuild.contentPath);
    pageDeps.insert(pageToBuild.templatePath);

    //creates anti-deps of readPath set
    std::set<Path> antiDepsOfReadPath;

    //starts read_and_process from templatePath
    if(read_and_process(pageToBuild.templatePath, antiDepsOfReadPath) > 0)
        return 1;

    //ensures @inputcontent was found inside template tree
    if(!contentAdded)
    {
        std::cout << "error: @inputcontent not found within template file " << pageToBuild.templatePath << " or any of its dependencies, content from " << pageToBuild.contentPath << " has not been inserted" << std::endl;
        return 1;
    }

    //makes sure page file exists
    pageToBuild.pagePath.ensurePathExists();

    //makes sure we can write to page file
    chmod(pageToBuild.pagePath.str().c_str(), 0644);

    //writes processed page to page file
    std::ofstream pageStream(pageToBuild.pagePath.str());
    pageStream << processedPage.str();
    pageStream.close();

    //makes sure user can't accidentally write to page file
    chmod(pageToBuild.pagePath.str().c_str(), 0444);

    //gets path for storing page information
    Path pageInfoPath = pageToBuild.pagePath.getInfoPath();

    //makes sure page info file exists
    pageInfoPath.ensurePathExists();

    //makes sure we can write to info file_
    chmod(pageInfoPath.str().c_str(), 0644);

    //writes page info file
    std::ofstream infoStream(pageInfoPath.str());
    infoStream << currentTime() << " " << currentDate() << std::endl;
    infoStream << this->pageToBuild << std::endl << std::endl;
    for(auto pageDep=pageDeps.begin(); pageDep != pageDeps.end(); pageDep++)
        infoStream << *pageDep << std::endl;
    infoStream.close();

    //makes sure user can't accidentally write to info file
    chmod(pageInfoPath.str().c_str(), 0444);

    std::cout << "page build successful" << std::endl;

    return 0;
};

//reads file whilst writing processed version to ofs
int PageBuilder::read_and_process(const Path &readPath, std::set<Path> antiDepsOfReadPath)
{
    //adds read path to anti dependencies of read path
    antiDepsOfReadPath.insert(readPath);

    std::ifstream ifs(readPath.str());
    std::string baseIndentAmount = indentAmount;
    int baseCodeBlockDepth = codeBlockDepth;

    int lineNo = 0,
        openCodeLineNo = 0;
    std::string inLine;
    while(getline(ifs, inLine))
    {
        lineNo++;

        if(lineNo > 1)
        {
            indentAmount = baseIndentAmount;
            processedPage << std::endl << baseIndentAmount;
        }

        for(size_t linePos=0; linePos<inLine.length();)
        {
            if(inLine[linePos] == '\\') //checks whether to escape
            {
                linePos++;
                /*
                    see http://dev.w3.org/html5/html-author/charref for
                    more character references than you could ever need!
                */
                switch(inLine[linePos])
                {
                    case '~':
                        processedPage << "˜";
                        linePos++;
                        indentAmount += std::string(7, ' ');
                        break;
                    case '!':
                        processedPage << "!";
                        linePos++;
                        indentAmount += std::string(6, ' ');
                        break;
                    case '@':
                        processedPage << "@";
                        linePos++;
                        indentAmount += std::string(8, ' ');
                        break;
                    case '#':
                        processedPage << "#";
                        linePos++;
                        indentAmount += std::string(5, ' ');
                        break;
                    /*case '$': //MUST HAVE MATHJAX HANDLE THIS
                        processedPage << "$";
                        linePos++;
                        indentAmount += string(8, ' ');
                        break;*/
                    case '%':
                        processedPage << "%";
                        linePos++;
                        indentAmount += std::string(8, ' ');
                        break;
                    case '^':
                        processedPage << "^";
                        linePos++;
                        indentAmount += std::string(5, ' ');
                        break;
                    case '&':
                        processedPage << "&";
                        linePos++;
                        indentAmount += std::string(5, ' ');
                        break;
                    case '*':
                        processedPage << "*";
                        linePos++;
                        indentAmount += std::string(5, ' ');
                        break;
                    case '?':
                        processedPage << "?";
                        linePos++;
                        indentAmount += std::string(7, ' ');
                        break;
                    case '<':
                        processedPage << "<";
                        linePos++;
                        indentAmount += std::string(4, ' ');
                        break;
                    case '>':
                        processedPage << ">";
                        linePos++;
                        indentAmount += std::string(4, ' ');
                        break;
                    default:
                        processedPage << "\\";
                        indentAmount += " ";
                }
            }
            else if(inLine[linePos] == '<') //checks about code blocks and html comments opening
            {
                //checks whether we're going down code block depth
                if(htmlCommentDepth == 0 && inLine.substr(linePos+1, 4) == "/pre")
                {
                    codeBlockDepth--;
                    if(codeBlockDepth < baseCodeBlockDepth)
                    {
                        std::cout << "error: " << readPath << ": line " << lineNo << ": </pre> close tag has no preceding <pre*> open tag." << std::endl << std::endl;
                        return 1;
                    }
                }

                //checks whether to escape <
                if(codeBlockDepth > 0)
                {
                    processedPage << "<";
                    indentAmount += std::string("<").length();
                }
                else
                {
                    processedPage << '<';
                    indentAmount += " ";
                }

                //checks whether we're going up code block depth
                if(htmlCommentDepth == 0 && inLine.substr(linePos+1, 3) == "pre")
                {
                    if(codeBlockDepth == baseCodeBlockDepth)
                        openCodeLineNo = lineNo;
                    codeBlockDepth++;
                }

                //checks whether we're going up html comment depth
                if(inLine.substr(linePos+1, 3) == "!--")
                    htmlCommentDepth++;

                linePos++;
            }
            else if(inLine[linePos] == '-') //checks about html comments closing
            {
                //checks whether we're going down html depth
                if(inLine.substr(linePos+1, 2) == "->")
                    htmlCommentDepth--;

                processedPage << '-';
                indentAmount += " ";
                linePos++;
            }
            else if(inLine[linePos] == '@') //checks for commands
            {
                if(linePos > 0 && inLine[linePos-1] == '\\')
                {
                    processedPage << '@';
                    indentAmount += " ";
                    linePos++;
                }
                else if(inLine.substr(linePos, 13) == "@inputcontent")
                {
                    contentAdded = 1;
                    std::string replaceText = "@input(" + pageToBuild.contentPath.str() + ")";
                    inLine.replace(linePos, 13, replaceText);
                }
                else if(inLine.substr(linePos, 7) == "@input(")
                {
                    linePos+=std::string("@input(").length();
                    std::string inputPathStr="";

                    for(; inLine[linePos] != ')'; linePos++)
                        if(inLine[linePos] != '"' && inLine[linePos] != '\'')
                            inputPathStr += inLine[linePos];
                    linePos++;

                    Path inputPath;
                    inputPath.set_file_path_from(inputPathStr);
                    pageDeps.insert(inputPath);

                    //ensures insert file exists
                    if(!std::ifstream(inputPathStr))
                    {
                        std::cout << "error: " << readPath << ": line " << lineNo << ": inputting file " << inputPath << " failed as path does not exist" << std::endl;
                        return 1;
                    }
                    //ensures insert file isn't an anti dep of read path
                    if(antiDepsOfReadPath.count(inputPath))
                    {
                        std::cout << "error: " << readPath << ": line " << lineNo << ": inputting file " << inputPath << " would result in an input loop" << std::endl;
                        return 1;
                    }

                    //adds insert file
                    if(read_and_process(inputPath, antiDepsOfReadPath) > 0)
                        return 1;
                    //indent amount updated inside read_and_process
                }
                else if(inLine.substr(linePos, 8) == "@pathto(")
                {
                    linePos+=std::string("@pathto(").length();
                    Name targetPageName="";

                    for(; inLine[linePos] != ')'; linePos++)
                        if(inLine[linePos] != '"' && inLine[linePos] != '\'')
                            targetPageName += inLine[linePos];
                    linePos++;

                    std::cout << targetPageName << std::endl;

                    //warns user if target targetPageName isn't being tracked by nsm
                    PageInfo targetPageInfo;
                    targetPageInfo.pageName = targetPageName;
                    if(!pages.count(targetPageInfo))
                    {
                        std::cout << "error: " << readPath << ": line " << lineNo << ": @pathto(" << targetPageName << ") failed, nsm not tracking " << targetPageName << std::endl;
                        return 1;
                    }

                    Path targetPath = pages.find(targetPageInfo)->pagePath;
                    //targetPath.set_file_path_from(targetPathStr);

                    std::cout << "finding path between " << pageToBuild.pagePath.dir << " to " << targetPath.dir << std::endl;

                    Path pathToTarget(pathBetween(pageToBuild.pagePath.dir, targetPath.dir), targetPath.file);

                    //adds path to target
                    processedPage << pathToTarget.str();
                    indentAmount += pathToTarget.str().length();
                }
                else if(inLine.substr(linePos, 10) == "@pagetitle")
                {
                    processedPage << pageToBuild.pageTitle.str;
                    indentAmount+=pageToBuild.pageTitle.str.length();
                    linePos+=std::string("@pagetitle").length();
                }
                else if(inLine.substr(linePos, 12) == "@currenttime")
                {
                    processedPage << dateTimeInfo.cTime;
                    indentAmount+=dateTimeInfo.cTime.length();
                    linePos+=std::string("@currenttime").length();
                }
                else if(inLine.substr(linePos, 12) == "@currentdate")
                {
                    processedPage << dateTimeInfo.cDate;
                    indentAmount+=dateTimeInfo.cDate.length();
                    linePos+=std::string("@currentdate").length();
                }
                else if(inLine.substr(linePos, 9) == "@timezone")
                {
                    processedPage << dateTimeInfo.cTimezone;
                    indentAmount+=dateTimeInfo.cTimezone.length();
                    linePos+=std::string("@timezone").length();
                }
                else if(inLine.substr(linePos, 15) == "@faviconinclude") //checks for favicon include
                {
                    linePos+=std::string("@faviconinclude(").length();
                    std::string faviconPathStr="";

                    for(; inLine[linePos] != ')'; linePos++)
                        if(inLine[linePos] != '"' && inLine[linePos] != '\'')
                            faviconPathStr += inLine[linePos];
                    linePos++;

                    //warns user if favicon file doesn't exist
                    if(!std::ifstream(faviconPathStr.c_str()))
                    {
                        std::cout << "warning: " << readPath << ": line " << lineNo << ": favicon file " << faviconPathStr << " does not exist" << std::endl;
                    }

                    Path faviconPath;
                    faviconPath.set_file_path_from(faviconPathStr);

                    Path pathToFavicon(pathBetween(pageToBuild.pagePath.dir, faviconPath.dir), faviconPath.file);

                    std::string faviconInclude="<link rel='icon' type='image/png' href='";
                    faviconInclude += pathToFavicon.str();
                    faviconInclude+="'>";

                    processedPage << faviconInclude;
                    indentAmount += faviconInclude.length();
                }
                else if(inLine.substr(linePos, 12) == "@cssinclude(") //checks for css includes
                {
                    linePos+=std::string("@cssinclude(").length();
                    std::string cssPathStr="";

                    for(; inLine[linePos] != ')'; linePos++)
                        if(inLine[linePos] != '"' && inLine[linePos] != '\'')
                            cssPathStr += inLine[linePos];
                    linePos++;

                    //warns user if css file doesn't exist
                    if(!std::ifstream(cssPathStr.c_str()))
                    {
                        std::cout << "warning: " << readPath << ": line " << lineNo << ": css file " << cssPathStr << "does not exist" << std::endl;
                    }

                    Path cssPath;
                    cssPath.set_file_path_from(cssPathStr);

                    Path pathToCSSFile(pathBetween(pageToBuild.pagePath.dir, cssPath.dir), cssPath.file);

                    std::string cssInclude="<link rel='stylesheet' type='text/css' href='";
                    cssInclude += pathToCSSFile.str();
                    cssInclude+="'>";

                    processedPage << cssInclude;
                    indentAmount += cssInclude.length();
                }
                else if(inLine.substr(linePos, 11) == "@jsinclude(") //checks for js includes
                {
                    linePos+=std::string("@jsinclude(").length();
                    std::string jsPathStr="";

                    for(; inLine[linePos] != ')'; linePos++)
                        if(inLine[linePos] != '"' && inLine[linePos] != '\'')
                            jsPathStr += inLine[linePos];
                    linePos++;

                    //warns user if js file doesn't exist
                    if(!std::ifstream(jsPathStr.c_str()))
                        std::cout << "warning: " << readPath << ": line " << lineNo << ": js file " << jsPathStr << "does not exist" << std::endl;

                    Path jsPath;
                    jsPath.set_file_path_from(jsPathStr);

                    Path pathToJSFile(pathBetween(pageToBuild.pagePath.dir, jsPath.dir), jsPath.file);

                    std::string jsInclude="<script type='text/javascript' src='";
                    jsInclude += pathToJSFile.str();
                    jsInclude+="'></script>";

                    processedPage << jsInclude;
                    indentAmount += jsInclude.length();
                }
                else
                {
                    processedPage << '@';
                    indentAmount += " ";
                    linePos++;
                }
            }
            else //regular character
            {
                if(inLine[linePos] == '\t')
                    indentAmount += '\t';
                else
                    indentAmount += ' ';
                processedPage << inLine[linePos++];
            }
        }
    }

    if(codeBlockDepth > baseCodeBlockDepth)
    {
        std::cout << "error: " << readPath << ": line " << openCodeLineNo << ": <pre*> open tag has no following </pre> close tag." << std::endl << std::endl;
        return 1;
    }

    ifs.close();

    return 0;
}
<commit_msg>added error when @pathto fails<commit_after>#include "PageBuilder.h"

PageBuilder::PageBuilder(const std::set<PageInfo> &Pages)
{
    pages = Pages;
}

int PageBuilder::build(const PageInfo &PageToBuild)
{
    pageToBuild = PageToBuild;

    std::cout << std::endl;

    //ensures content and template files exist
    if(!std::ifstream(pageToBuild.contentPath.str()))
    {
        std::cout << "error: cannot build " << pageToBuild.pagePath << " as content file " << pageToBuild.contentPath << " does not exist" << std::endl;
        return 1;
    }
    if(!std::ifstream(pageToBuild.templatePath.str()))
    {
        std::cout << "error: cannot build " << pageToBuild.pagePath << " as template file " << pageToBuild.templatePath << " does not exist." << std::endl;
        return 1;
    }

    std::cout << "building page " << pageToBuild.pagePath << std::endl;

    //makes sure variables are at default values
    codeBlockDepth = htmlCommentDepth = 0;
    indentAmount = "";
    contentAdded = 0;
    processedPage.clear();
    processedPage.str(std::string());
    pageDeps.clear();
    contentAdded = 0;

    //adds content and template paths to dependencies
    pageDeps.insert(pageToBuild.contentPath);
    pageDeps.insert(pageToBuild.templatePath);

    //creates anti-deps of readPath set
    std::set<Path> antiDepsOfReadPath;

    //starts read_and_process from templatePath
    if(read_and_process(pageToBuild.templatePath, antiDepsOfReadPath) > 0)
        return 1;

    //ensures @inputcontent was found inside template tree
    if(!contentAdded)
    {
        std::cout << "error: @inputcontent not found within template file " << pageToBuild.templatePath << " or any of its dependencies, content from " << pageToBuild.contentPath << " has not been inserted" << std::endl;
        return 1;
    }

    //makes sure page file exists
    pageToBuild.pagePath.ensurePathExists();

    //makes sure we can write to page file
    chmod(pageToBuild.pagePath.str().c_str(), 0644);

    //writes processed page to page file
    std::ofstream pageStream(pageToBuild.pagePath.str());
    pageStream << processedPage.str();
    pageStream.close();

    //makes sure user can't accidentally write to page file
    chmod(pageToBuild.pagePath.str().c_str(), 0444);

    //gets path for storing page information
    Path pageInfoPath = pageToBuild.pagePath.getInfoPath();

    //makes sure page info file exists
    pageInfoPath.ensurePathExists();

    //makes sure we can write to info file_
    chmod(pageInfoPath.str().c_str(), 0644);

    //writes page info file
    std::ofstream infoStream(pageInfoPath.str());
    infoStream << currentTime() << " " << currentDate() << std::endl;
    infoStream << this->pageToBuild << std::endl << std::endl;
    for(auto pageDep=pageDeps.begin(); pageDep != pageDeps.end(); pageDep++)
        infoStream << *pageDep << std::endl;
    infoStream.close();

    //makes sure user can't accidentally write to info file
    chmod(pageInfoPath.str().c_str(), 0444);

    std::cout << "page build successful" << std::endl;

    return 0;
};

//reads file whilst writing processed version to ofs
int PageBuilder::read_and_process(const Path &readPath, std::set<Path> antiDepsOfReadPath)
{
    //adds read path to anti dependencies of read path
    antiDepsOfReadPath.insert(readPath);

    std::ifstream ifs(readPath.str());
    std::string baseIndentAmount = indentAmount;
    int baseCodeBlockDepth = codeBlockDepth;

    int lineNo = 0,
        openCodeLineNo = 0;
    std::string inLine;
    while(getline(ifs, inLine))
    {
        lineNo++;

        if(lineNo > 1)
        {
            indentAmount = baseIndentAmount;
            processedPage << std::endl << baseIndentAmount;
        }

        for(size_t linePos=0; linePos<inLine.length();)
        {
            if(inLine[linePos] == '\\') //checks whether to escape
            {
                linePos++;
                /*
                    see http://dev.w3.org/html5/html-author/charref for
                    more character references than you could ever need!
                */
                switch(inLine[linePos])
                {
                    case '~':
                        processedPage << "˜";
                        linePos++;
                        indentAmount += std::string(7, ' ');
                        break;
                    case '!':
                        processedPage << "!";
                        linePos++;
                        indentAmount += std::string(6, ' ');
                        break;
                    case '@':
                        processedPage << "@";
                        linePos++;
                        indentAmount += std::string(8, ' ');
                        break;
                    case '#':
                        processedPage << "#";
                        linePos++;
                        indentAmount += std::string(5, ' ');
                        break;
                    /*case '$': //MUST HAVE MATHJAX HANDLE THIS
                        processedPage << "$";
                        linePos++;
                        indentAmount += string(8, ' ');
                        break;*/
                    case '%':
                        processedPage << "%";
                        linePos++;
                        indentAmount += std::string(8, ' ');
                        break;
                    case '^':
                        processedPage << "^";
                        linePos++;
                        indentAmount += std::string(5, ' ');
                        break;
                    case '&':
                        processedPage << "&";
                        linePos++;
                        indentAmount += std::string(5, ' ');
                        break;
                    case '*':
                        processedPage << "*";
                        linePos++;
                        indentAmount += std::string(5, ' ');
                        break;
                    case '?':
                        processedPage << "?";
                        linePos++;
                        indentAmount += std::string(7, ' ');
                        break;
                    case '<':
                        processedPage << "<";
                        linePos++;
                        indentAmount += std::string(4, ' ');
                        break;
                    case '>':
                        processedPage << ">";
                        linePos++;
                        indentAmount += std::string(4, ' ');
                        break;
                    default:
                        processedPage << "\\";
                        indentAmount += " ";
                }
            }
            else if(inLine[linePos] == '<') //checks about code blocks and html comments opening
            {
                //checks whether we're going down code block depth
                if(htmlCommentDepth == 0 && inLine.substr(linePos+1, 4) == "/pre")
                {
                    codeBlockDepth--;
                    if(codeBlockDepth < baseCodeBlockDepth)
                    {
                        std::cout << "error: " << readPath << ": line " << lineNo << ": </pre> close tag has no preceding <pre*> open tag." << std::endl << std::endl;
                        return 1;
                    }
                }

                //checks whether to escape <
                if(codeBlockDepth > 0)
                {
                    processedPage << "<";
                    indentAmount += std::string("<").length();
                }
                else
                {
                    processedPage << '<';
                    indentAmount += " ";
                }

                //checks whether we're going up code block depth
                if(htmlCommentDepth == 0 && inLine.substr(linePos+1, 3) == "pre")
                {
                    if(codeBlockDepth == baseCodeBlockDepth)
                        openCodeLineNo = lineNo;
                    codeBlockDepth++;
                }

                //checks whether we're going up html comment depth
                if(inLine.substr(linePos+1, 3) == "!--")
                    htmlCommentDepth++;

                linePos++;
            }
            else if(inLine[linePos] == '-') //checks about html comments closing
            {
                //checks whether we're going down html depth
                if(inLine.substr(linePos+1, 2) == "->")
                    htmlCommentDepth--;

                processedPage << '-';
                indentAmount += " ";
                linePos++;
            }
            else if(inLine[linePos] == '@') //checks for commands
            {
                if(linePos > 0 && inLine[linePos-1] == '\\')
                {
                    processedPage << '@';
                    indentAmount += " ";
                    linePos++;
                }
                else if(inLine.substr(linePos, 13) == "@inputcontent")
                {
                    contentAdded = 1;
                    std::string replaceText = "@input(" + pageToBuild.contentPath.str() + ")";
                    inLine.replace(linePos, 13, replaceText);
                }
                else if(inLine.substr(linePos, 7) == "@input(")
                {
                    linePos+=std::string("@input(").length();
                    std::string inputPathStr="";

                    for(; inLine[linePos] != ')'; linePos++)
                        if(inLine[linePos] != '"' && inLine[linePos] != '\'')
                            inputPathStr += inLine[linePos];
                    linePos++;

                    Path inputPath;
                    inputPath.set_file_path_from(inputPathStr);
                    pageDeps.insert(inputPath);

                    //ensures insert file exists
                    if(!std::ifstream(inputPathStr))
                    {
                        std::cout << "error: " << readPath << ": line " << lineNo << ": inputting file " << inputPath << " failed as path does not exist" << std::endl;
                        return 1;
                    }
                    //ensures insert file isn't an anti dep of read path
                    if(antiDepsOfReadPath.count(inputPath))
                    {
                        std::cout << "error: " << readPath << ": line " << lineNo << ": inputting file " << inputPath << " would result in an input loop" << std::endl;
                        return 1;
                    }

                    //adds insert file
                    if(read_and_process(inputPath, antiDepsOfReadPath) > 0)
                        return 1;
                    //indent amount updated inside read_and_process
                }
                else if(inLine.substr(linePos, 8) == "@pathto(")
                {
                    linePos+=std::string("@pathto(").length();
                    Name targetPageName="";

                    for(; inLine[linePos] != ')'; linePos++)
                        if(inLine[linePos] != '"' && inLine[linePos] != '\'')
                            targetPageName += inLine[linePos];
                    linePos++;

                    //warns user if target targetPageName isn't being tracked by nsm
                    PageInfo targetPageInfo;
                    targetPageInfo.pageName = targetPageName;
                    if(!pages.count(targetPageInfo))
                    {
                        std::cout << "error: " << readPath << ": line " << lineNo << ": @pathto(" << targetPageName << ") failed, nsm not tracking " << targetPageName << std::endl;
                        return 1;
                    }

                    Path targetPath = pages.find(targetPageInfo)->pagePath;
                    //targetPath.set_file_path_from(targetPathStr);

                    Path pathToTarget(pathBetween(pageToBuild.pagePath.dir, targetPath.dir), targetPath.file);

                    //adds path to target
                    processedPage << pathToTarget.str();
                    indentAmount += pathToTarget.str().length();
                }
                else if(inLine.substr(linePos, 10) == "@pagetitle")
                {
                    processedPage << pageToBuild.pageTitle.str;
                    indentAmount+=pageToBuild.pageTitle.str.length();
                    linePos+=std::string("@pagetitle").length();
                }
                else if(inLine.substr(linePos, 12) == "@currenttime")
                {
                    processedPage << dateTimeInfo.cTime;
                    indentAmount+=dateTimeInfo.cTime.length();
                    linePos+=std::string("@currenttime").length();
                }
                else if(inLine.substr(linePos, 12) == "@currentdate")
                {
                    processedPage << dateTimeInfo.cDate;
                    indentAmount+=dateTimeInfo.cDate.length();
                    linePos+=std::string("@currentdate").length();
                }
                else if(inLine.substr(linePos, 9) == "@timezone")
                {
                    processedPage << dateTimeInfo.cTimezone;
                    indentAmount+=dateTimeInfo.cTimezone.length();
                    linePos+=std::string("@timezone").length();
                }
                else if(inLine.substr(linePos, 15) == "@faviconinclude") //checks for favicon include
                {
                    linePos+=std::string("@faviconinclude(").length();
                    std::string faviconPathStr="";

                    for(; inLine[linePos] != ')'; linePos++)
                        if(inLine[linePos] != '"' && inLine[linePos] != '\'')
                            faviconPathStr += inLine[linePos];
                    linePos++;

                    //warns user if favicon file doesn't exist
                    if(!std::ifstream(faviconPathStr.c_str()))
                    {
                        std::cout << "warning: " << readPath << ": line " << lineNo << ": favicon file " << faviconPathStr << " does not exist" << std::endl;
                    }

                    Path faviconPath;
                    faviconPath.set_file_path_from(faviconPathStr);

                    Path pathToFavicon(pathBetween(pageToBuild.pagePath.dir, faviconPath.dir), faviconPath.file);

                    std::string faviconInclude="<link rel='icon' type='image/png' href='";
                    faviconInclude += pathToFavicon.str();
                    faviconInclude+="'>";

                    processedPage << faviconInclude;
                    indentAmount += faviconInclude.length();
                }
                else if(inLine.substr(linePos, 12) == "@cssinclude(") //checks for css includes
                {
                    linePos+=std::string("@cssinclude(").length();
                    std::string cssPathStr="";

                    for(; inLine[linePos] != ')'; linePos++)
                        if(inLine[linePos] != '"' && inLine[linePos] != '\'')
                            cssPathStr += inLine[linePos];
                    linePos++;

                    //warns user if css file doesn't exist
                    if(!std::ifstream(cssPathStr.c_str()))
                    {
                        std::cout << "warning: " << readPath << ": line " << lineNo << ": css file " << cssPathStr << "does not exist" << std::endl;
                    }

                    Path cssPath;
                    cssPath.set_file_path_from(cssPathStr);

                    Path pathToCSSFile(pathBetween(pageToBuild.pagePath.dir, cssPath.dir), cssPath.file);

                    std::string cssInclude="<link rel='stylesheet' type='text/css' href='";
                    cssInclude += pathToCSSFile.str();
                    cssInclude+="'>";

                    processedPage << cssInclude;
                    indentAmount += cssInclude.length();
                }
                else if(inLine.substr(linePos, 11) == "@jsinclude(") //checks for js includes
                {
                    linePos+=std::string("@jsinclude(").length();
                    std::string jsPathStr="";

                    for(; inLine[linePos] != ')'; linePos++)
                        if(inLine[linePos] != '"' && inLine[linePos] != '\'')
                            jsPathStr += inLine[linePos];
                    linePos++;

                    //warns user if js file doesn't exist
                    if(!std::ifstream(jsPathStr.c_str()))
                        std::cout << "warning: " << readPath << ": line " << lineNo << ": js file " << jsPathStr << "does not exist" << std::endl;

                    Path jsPath;
                    jsPath.set_file_path_from(jsPathStr);

                    Path pathToJSFile(pathBetween(pageToBuild.pagePath.dir, jsPath.dir), jsPath.file);

                    std::string jsInclude="<script type='text/javascript' src='";
                    jsInclude += pathToJSFile.str();
                    jsInclude+="'></script>";

                    processedPage << jsInclude;
                    indentAmount += jsInclude.length();
                }
                else
                {
                    processedPage << '@';
                    indentAmount += " ";
                    linePos++;
                }
            }
            else //regular character
            {
                if(inLine[linePos] == '\t')
                    indentAmount += '\t';
                else
                    indentAmount += ' ';
                processedPage << inLine[linePos++];
            }
        }
    }

    if(codeBlockDepth > baseCodeBlockDepth)
    {
        std::cout << "error: " << readPath << ": line " << openCodeLineNo << ": <pre*> open tag has no following </pre> close tag." << std::endl << std::endl;
        return 1;
    }

    ifs.close();

    return 0;
}
<|endoftext|>


<commit_before>/*
 *  Copyright (c) 2016, Facebook, Inc.
 *  All rights reserved.
 *
 *  This source code is licensed under the BSD-style license found in the
 *  LICENSE file in the root directory of this source tree. An additional grant
 *  of patent rights can be found in the PATENTS file in the same directory.
 *
 */
#include <gtest/gtest.h>

#include <unistd.h>

#include <folly/Exception.h>
#include <folly/File.h>
#include <folly/io/async/EventBase.h>
#include <folly/Random.h>

#include "bistro/bistro/processes/AsyncReadPipe.h"
#include "bistro/bistro/processes/tests/utils.h"

using namespace facebook::bistro;

/**
 * Reads lines from a pipe via asyncReadPipe() and readPipeLinesCallback().
 *
 * Writes message to a pipe in randomly sized chunks. Randomly alternates
 * between reading and writing (so that "read read write", and "write write
 * read" patterns are fairly probable, as are strict alternations).
 *
 * Checks that the lines conform to: a line always contains exactly one
 * delimiter (at the end of the line), except for EOF, in which case the
 * line has no delimiter.
 *
 * Checks that the concatenation of the lines reconstructs the message.
 */
void checkPipeRead(const std::string& message) {
  folly::File read_pipe, write_pipe;
  makePipe(&read_pipe, &write_pipe);

  size_t cursor = 0;
  auto write_chunk_fn = [&]() {
    auto bytes_left = message.size() - cursor;
    if (!bytes_left) {
      write_pipe.close();  // idempotent
      return;
    }
    auto to_write = folly::Random::rand32(1, bytes_left + 1);
    auto written =
      folly::writeNoInt(write_pipe.fd(), message.data() + cursor, to_write);
    folly::checkUnixError(written, "write");
    LOG(INFO) << "wrote: '" << folly::StringPiece(
      message.data() + cursor, message.data() + cursor + written
    ) << "'";
    cursor += written;
  };

  folly::EventBase evb;
  int read_fd = read_pipe.fd();
  bool saw_eof = false;  // Ensure the callback sees EOF exactly once
  std::string read_message;  // Try to reconstruct the message
  auto closed = asyncReadPipe(
    &evb,
    std::move(read_pipe),
    readPipeLinesCallback([&](AsyncReadPipe*, folly::StringPiece s) {
      EXPECT_FALSE(saw_eof);  // Saw EOF at most once
      read_message.append(s.begin(), s.end());
      if (!s.empty()) {
        LOG(INFO) << read_fd << " read: '" << s << "'";
      }
      if (s.empty() || s.back() != '\n') {
        EXPECT_EQ(std::string::npos, folly::qfind(s, '\n'));  // No EOL at all
        LOG(INFO) << read_fd << " - EOF";
        saw_eof = true;
      } else {
        EXPECT_EQ(s.size() - 1, folly::qfind(s, '\n'));  // EOL only at the end
      }
    })
  )->pipeClosed();

  while (!closed.isReady()) {
    if (folly::Random::oneIn(2)) {
      write_chunk_fn();
    }
    if (folly::Random::oneIn(2)) {
      evb.loopOnce(EVLOOP_NONBLOCK);
    }
  }
  EXPECT_TRUE(saw_eof);
  EXPECT_EQ(message, read_message);
}

TEST(TestAsyncReadPipe, SimpleRead) {
  // Repeat the test a few times since it is randomized.
  for (size_t i = 0; i < 50; ++i) {
    checkPipeRead("");
    checkPipeRead("bunnies eat carrots and cabbage");
    checkPipeRead("\n\nbunnies\n\n\neat\ncarrots\nand\ncabbage\n\n");
  }
}

namespace { class TestError : public std::exception {}; }

TEST(TestAsyncReadPipe, CallbackException) {
  folly::File read_pipe;
  makePipe(&read_pipe, nullptr);

  folly::EventBase evb;
  auto closed = asyncReadPipe(
    &evb,
    std::move(read_pipe),
    [&](AsyncReadPipe*) -> bool { throw TestError(); }
  )->pipeClosed();
  while (!closed.isReady()) { evb.loop(); }
  EXPECT_THROW(closed.get(), TestError);
}

TEST(TestAsyncReadPipe, PauseResume) {
  folly::File read_pipe, write_pipe;
  makePipe(&read_pipe, &write_pipe);

  size_t lines_before_pause = 3;
  folly::EventBase evb;
  auto pipe = asyncReadPipe(
    &evb,
    std::move(read_pipe),
    readPipeLinesCallback([&](AsyncReadPipe* pipe, folly::StringPiece s) {
      if (!s.empty()) {
        LOG(INFO) << "Read: '" << s << "'";
        --lines_before_pause;
        if (lines_before_pause == 0) {
          pipe->pause();
        }
      }
    }, 0, '\n', 1)  // 1-char buffer to ensure that pause is "instant"
  );
  auto closed = pipe->pipeClosed();

  auto write_fn = [&](const char* msg) {
    // Will deadlock if the pipe buffer fills up, but I'll take my chances.
    folly::writeFull(write_pipe.fd(), msg, strlen(msg));
  };

  evb.loopOnce(EVLOOP_NONBLOCK);  // No data, no action
  EXPECT_EQ(3, lines_before_pause);

  // The first 3 are consumed, and then we pause.
  for (int i = 0; i < 6; ++i) {
    write_fn("a\n");
    evb.loopOnce(EVLOOP_NONBLOCK);
    EXPECT_EQ((i < 3) ? (2 - i) : 0, lines_before_pause);
  }

  // Consume all the lines, and check that no more events fire.
  lines_before_pause = 5;
  pipe->resume();
  EXPECT_EQ(5, lines_before_pause);  // Doesn't fire without the EventBase
  evb.loopOnce(EVLOOP_NONBLOCK);
  EXPECT_EQ(2, lines_before_pause);  // All 3 blocked lines get consumed
  evb.loopOnce(EVLOOP_NONBLOCK);  // Nothing more to read yet.
  write_fn("b\nc\nd\nf\ng\n");
  EXPECT_EQ(2, lines_before_pause);
  evb.loopOnce(EVLOOP_NONBLOCK);
  EXPECT_EQ(0, lines_before_pause);  // 2 consumed, 3 to go

  lines_before_pause = 1;
  pipe->resume();
  EXPECT_EQ(1, lines_before_pause);
  evb.loopOnce(EVLOOP_NONBLOCK);
  EXPECT_EQ(0, lines_before_pause);  // 1 consumed, 2 to go

  lines_before_pause = 2;
  pipe->resume();
  EXPECT_EQ(2, lines_before_pause);
  evb.loopOnce(EVLOOP_NONBLOCK);
  EXPECT_EQ(0, lines_before_pause);  // All consumed

  EXPECT_FALSE(closed.isReady());
  write_pipe.close();
  EXPECT_FALSE(closed.isReady());  // The EventBase didn't run yet
  evb.loopOnce(EVLOOP_NONBLOCK);
  EXPECT_FALSE(closed.isReady());  // The pipe is paused, so we didn't know
  pipe->resume();
  evb.loopOnce(EVLOOP_NONBLOCK);
  EXPECT_TRUE(closed.isReady());  // Now we know
  EXPECT_EQ(0, lines_before_pause);  // No more lines were seen
}

TEST(TestAsyncReadPipe, ExternalClose) {
  folly::File read_pipe, write_pipe;
  makePipe(&read_pipe, &write_pipe);

  folly::EventBase evb;
  size_t num_lines = 0;
  auto pipe = asyncReadPipe(
      &evb,
      std::move(read_pipe),
      readPipeLinesCallback(
          [&](AsyncReadPipe* /*pipe*/, folly::StringPiece s) {
            if (!s.empty()) {
              LOG(INFO) << "Read: '" << s << "'";
              ++num_lines;
            }
          },
          0,
          '\n',
          1) // 1-char buffer to ensure that close is "instant"
      );
  auto closed = pipe->pipeClosed();

  evb.loopOnce(EVLOOP_NONBLOCK);  // No data, no action
  EXPECT_EQ(0, num_lines);
  EXPECT_FALSE(closed.isReady());

  auto write_fn = [&](const char* msg) {
    // Will deadlock if the pipe buffer fills up, but I'll take my chances.
    folly::writeFull(write_pipe.fd(), msg, strlen(msg));
  };

  // Feed through some lines without closing the pipe
  const int kMaxLines = 6;
  for (int i = 0; i < kMaxLines; ++i) {
    write_fn("a\n");
    evb.loopOnce(EVLOOP_NONBLOCK);
    EXPECT_EQ(i + 1, num_lines);
    EXPECT_FALSE(closed.isReady());
  }
  // These writes will get lost since we're synchronously closing the pipe.
  write_fn("b\n");
  write_fn("c\n");
  write_fn("d\n");
  pipe->close();
  EXPECT_TRUE(closed.isReady());

  // Nothing more happens on this event base, and we're the pipe's sole owner.
  evb.loopOnce(EVLOOP_NONBLOCK);
  EXPECT_EQ(kMaxLines, num_lines);
  EXPECT_EQ(1, pipe.use_count());
}
<commit_msg>use rvalue-qual Future::get(): pass 5<commit_after>/*
 *  Copyright (c) 2016-present, Facebook, Inc.
 *  All rights reserved.
 *
 *  This source code is licensed under the BSD-style license found in the
 *  LICENSE file in the root directory of this source tree. An additional grant
 *  of patent rights can be found in the PATENTS file in the same directory.
 *
 */
#include <gtest/gtest.h>

#include <unistd.h>

#include <folly/Exception.h>
#include <folly/File.h>
#include <folly/io/async/EventBase.h>
#include <folly/Random.h>

#include "bistro/bistro/processes/AsyncReadPipe.h"
#include "bistro/bistro/processes/tests/utils.h"

using namespace facebook::bistro;

/**
 * Reads lines from a pipe via asyncReadPipe() and readPipeLinesCallback().
 *
 * Writes message to a pipe in randomly sized chunks. Randomly alternates
 * between reading and writing (so that "read read write", and "write write
 * read" patterns are fairly probable, as are strict alternations).
 *
 * Checks that the lines conform to: a line always contains exactly one
 * delimiter (at the end of the line), except for EOF, in which case the
 * line has no delimiter.
 *
 * Checks that the concatenation of the lines reconstructs the message.
 */
void checkPipeRead(const std::string& message) {
  folly::File read_pipe, write_pipe;
  makePipe(&read_pipe, &write_pipe);

  size_t cursor = 0;
  auto write_chunk_fn = [&]() {
    auto bytes_left = message.size() - cursor;
    if (!bytes_left) {
      write_pipe.close();  // idempotent
      return;
    }
    auto to_write = folly::Random::rand32(1, bytes_left + 1);
    auto written =
      folly::writeNoInt(write_pipe.fd(), message.data() + cursor, to_write);
    folly::checkUnixError(written, "write");
    LOG(INFO) << "wrote: '" << folly::StringPiece(
      message.data() + cursor, message.data() + cursor + written
    ) << "'";
    cursor += written;
  };

  folly::EventBase evb;
  int read_fd = read_pipe.fd();
  bool saw_eof = false;  // Ensure the callback sees EOF exactly once
  std::string read_message;  // Try to reconstruct the message
  auto closed = asyncReadPipe(
    &evb,
    std::move(read_pipe),
    readPipeLinesCallback([&](AsyncReadPipe*, folly::StringPiece s) {
      EXPECT_FALSE(saw_eof);  // Saw EOF at most once
      read_message.append(s.begin(), s.end());
      if (!s.empty()) {
        LOG(INFO) << read_fd << " read: '" << s << "'";
      }
      if (s.empty() || s.back() != '\n') {
        EXPECT_EQ(std::string::npos, folly::qfind(s, '\n'));  // No EOL at all
        LOG(INFO) << read_fd << " - EOF";
        saw_eof = true;
      } else {
        EXPECT_EQ(s.size() - 1, folly::qfind(s, '\n'));  // EOL only at the end
      }
    })
  )->pipeClosed();

  while (!closed.isReady()) {
    if (folly::Random::oneIn(2)) {
      write_chunk_fn();
    }
    if (folly::Random::oneIn(2)) {
      evb.loopOnce(EVLOOP_NONBLOCK);
    }
  }
  EXPECT_TRUE(saw_eof);
  EXPECT_EQ(message, read_message);
}

TEST(TestAsyncReadPipe, SimpleRead) {
  // Repeat the test a few times since it is randomized.
  for (size_t i = 0; i < 50; ++i) {
    checkPipeRead("");
    checkPipeRead("bunnies eat carrots and cabbage");
    checkPipeRead("\n\nbunnies\n\n\neat\ncarrots\nand\ncabbage\n\n");
  }
}

namespace { class TestError : public std::exception {}; }

TEST(TestAsyncReadPipe, CallbackException) {
  folly::File read_pipe;
  makePipe(&read_pipe, nullptr);

  folly::EventBase evb;
  auto closed = asyncReadPipe(
    &evb,
    std::move(read_pipe),
    [&](AsyncReadPipe*) -> bool { throw TestError(); }
  )->pipeClosed();
  while (!closed.isReady()) { evb.loop(); }
  EXPECT_THROW(std::move(closed).get(), TestError);
}

TEST(TestAsyncReadPipe, PauseResume) {
  folly::File read_pipe, write_pipe;
  makePipe(&read_pipe, &write_pipe);

  size_t lines_before_pause = 3;
  folly::EventBase evb;
  auto pipe = asyncReadPipe(
    &evb,
    std::move(read_pipe),
    readPipeLinesCallback([&](AsyncReadPipe* pipe, folly::StringPiece s) {
      if (!s.empty()) {
        LOG(INFO) << "Read: '" << s << "'";
        --lines_before_pause;
        if (lines_before_pause == 0) {
          pipe->pause();
        }
      }
    }, 0, '\n', 1)  // 1-char buffer to ensure that pause is "instant"
  );
  auto closed = pipe->pipeClosed();

  auto write_fn = [&](const char* msg) {
    // Will deadlock if the pipe buffer fills up, but I'll take my chances.
    folly::writeFull(write_pipe.fd(), msg, strlen(msg));
  };

  evb.loopOnce(EVLOOP_NONBLOCK);  // No data, no action
  EXPECT_EQ(3, lines_before_pause);

  // The first 3 are consumed, and then we pause.
  for (int i = 0; i < 6; ++i) {
    write_fn("a\n");
    evb.loopOnce(EVLOOP_NONBLOCK);
    EXPECT_EQ((i < 3) ? (2 - i) : 0, lines_before_pause);
  }

  // Consume all the lines, and check that no more events fire.
  lines_before_pause = 5;
  pipe->resume();
  EXPECT_EQ(5, lines_before_pause);  // Doesn't fire without the EventBase
  evb.loopOnce(EVLOOP_NONBLOCK);
  EXPECT_EQ(2, lines_before_pause);  // All 3 blocked lines get consumed
  evb.loopOnce(EVLOOP_NONBLOCK);  // Nothing more to read yet.
  write_fn("b\nc\nd\nf\ng\n");
  EXPECT_EQ(2, lines_before_pause);
  evb.loopOnce(EVLOOP_NONBLOCK);
  EXPECT_EQ(0, lines_before_pause);  // 2 consumed, 3 to go

  lines_before_pause = 1;
  pipe->resume();
  EXPECT_EQ(1, lines_before_pause);
  evb.loopOnce(EVLOOP_NONBLOCK);
  EXPECT_EQ(0, lines_before_pause);  // 1 consumed, 2 to go

  lines_before_pause = 2;
  pipe->resume();
  EXPECT_EQ(2, lines_before_pause);
  evb.loopOnce(EVLOOP_NONBLOCK);
  EXPECT_EQ(0, lines_before_pause);  // All consumed

  EXPECT_FALSE(closed.isReady());
  write_pipe.close();
  EXPECT_FALSE(closed.isReady());  // The EventBase didn't run yet
  evb.loopOnce(EVLOOP_NONBLOCK);
  EXPECT_FALSE(closed.isReady());  // The pipe is paused, so we didn't know
  pipe->resume();
  evb.loopOnce(EVLOOP_NONBLOCK);
  EXPECT_TRUE(closed.isReady());  // Now we know
  EXPECT_EQ(0, lines_before_pause);  // No more lines were seen
}

TEST(TestAsyncReadPipe, ExternalClose) {
  folly::File read_pipe, write_pipe;
  makePipe(&read_pipe, &write_pipe);

  folly::EventBase evb;
  size_t num_lines = 0;
  auto pipe = asyncReadPipe(
      &evb,
      std::move(read_pipe),
      readPipeLinesCallback(
          [&](AsyncReadPipe* /*pipe*/, folly::StringPiece s) {
            if (!s.empty()) {
              LOG(INFO) << "Read: '" << s << "'";
              ++num_lines;
            }
          },
          0,
          '\n',
          1) // 1-char buffer to ensure that close is "instant"
      );
  auto closed = pipe->pipeClosed();

  evb.loopOnce(EVLOOP_NONBLOCK);  // No data, no action
  EXPECT_EQ(0, num_lines);
  EXPECT_FALSE(closed.isReady());

  auto write_fn = [&](const char* msg) {
    // Will deadlock if the pipe buffer fills up, but I'll take my chances.
    folly::writeFull(write_pipe.fd(), msg, strlen(msg));
  };

  // Feed through some lines without closing the pipe
  const int kMaxLines = 6;
  for (int i = 0; i < kMaxLines; ++i) {
    write_fn("a\n");
    evb.loopOnce(EVLOOP_NONBLOCK);
    EXPECT_EQ(i + 1, num_lines);
    EXPECT_FALSE(closed.isReady());
  }
  // These writes will get lost since we're synchronously closing the pipe.
  write_fn("b\n");
  write_fn("c\n");
  write_fn("d\n");
  pipe->close();
  EXPECT_TRUE(closed.isReady());

  // Nothing more happens on this event base, and we're the pipe's sole owner.
  evb.loopOnce(EVLOOP_NONBLOCK);
  EXPECT_EQ(kMaxLines, num_lines);
  EXPECT_EQ(1, pipe.use_count());
}
<|endoftext|>