<commit_before>#define DEBUG 1
/**
 * File    : D.cpp
 * Author  : Kazune Takahashi
 * Created : 8/11/2019, 12:04:39 AM
 * Powered by Visual Studio Code
 */
#include <iostream>
#include <iomanip>
#include <algorithm>
#include <vector>
#include <string>
#include <complex>
#include <tuple>
#include <queue>
#include <stack>
#include <map>
#include <set>
#include <unordered_map>
#include <unordered_set>
#include <bitset>
#include <functional>
#include <random>
#include <chrono>
#include <cctype>
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstdlib>
using namespace std;
#define maxs(x, y) (x = max(x, y))
#define mins(x, y) (x = min(x, y))
using ll = long long;
class mint
{
public:
  static ll MOD;
  ll x;
  mint() : x(0) {}
  mint(ll x) : x(x % MOD) {}
  mint operator-() const { return x ? MOD - x : 0; }
  mint &operator+=(const mint &a)
  {
    if ((x += a.x) >= MOD)
    {
      x -= MOD;
    }
    return *this;
  }
  mint &operator-=(const mint &a) { return *this += -a; }
  mint &operator*=(const mint &a)
  {
    (x *= a.x) %= MOD;
    return *this;
  }
  mint &operator/=(const mint &a)
  {
    mint b{a};
    return *this *= b.power(MOD - 2);
  }
  mint operator+(const mint &a) const { return mint(*this) += a; }
  mint operator-(const mint &a) const { return mint(*this) -= a; }
  mint operator*(const mint &a) const { return mint(*this) *= a; }
  mint operator/(const mint &a) const { return mint(*this) /= a; }
  bool operator<(const mint &a) const { return x < a.x; }
  bool operator==(const mint &a) const { return x == a.x; }
  const mint power(ll N)
  {
    if (N == 0)
    {
      return 1;
    }
    else if (N % 2 == 1)
    {
      return *this * power(N - 1);
    }
    else
    {
      mint half = power(N / 2);
      return half * half;
    }
  }
};
ll mint::MOD = 1e9 + 7;
istream &operator>>(istream &stream, mint &a) { return stream >> a.x; }
ostream &operator<<(ostream &stream, const mint &a) { return stream << a.x; }
class combination
{
public:
  vector<mint> inv, fact, factinv;
  static int MAX_SIZE;
  combination() : inv(MAX_SIZE), fact(MAX_SIZE), factinv(MAX_SIZE)
  {
    inv[1] = 1;
    for (auto i = 2; i < MAX_SIZE; i++)
    {
      inv[i] = (-inv[mint::MOD % i]) * (mint::MOD / i);
    }
    fact[0] = factinv[0] = 1;
    for (auto i = 1; i < MAX_SIZE; i++)
    {
      fact[i] = mint(i) * fact[i - 1];
      factinv[i] = inv[i] * factinv[i - 1];
    }
  }
  mint operator()(int n, int k)
  {
    if (n >= 0 && k >= 0 && n - k >= 0)
    {
      return fact[n] * factinv[k] * factinv[n - k];
    }
    return 0;
  }
};
int combination::MAX_SIZE = 3000010;
ll gcd(ll x, ll y) { return y ? gcd(y, x % y) : x; }
// constexpr double epsilon = 1e-10;
// constexpr ll infty = 1000000000000000LL;
// constexpr int dx[4] = {1, 0, -1, 0};
// constexpr int dy[4] = {0, 1, 0, -1};
void Yes()
{
  cout << "Yes" << endl;
  exit(0);
}
void No()
{
  cout << "No" << endl;
  exit(0);
}

using work = tuple<ll, ll>;

int main()
{
  int N, M;
  cin >> N >> M;
  vector<vector<ll>> V(M, vector<ll>{});
  for (auto i = 0; i < N; i++)
  {
    int A;
    ll B;
    cin >> A >> B;
    int t = M - A;
    if (t >= 0)
    V[t].push_back(B);
  }
  priority_queue<ll> Q;
  ll ans = 0;
  for (auto i = M - 1; i >= 0; i--)
  {
    for (auto x : V[i])
    {
      Q.push(x);
    }
    if (!Q.empty())
    {
      ans += Q.top();
      Q.pop();
    }
  }
  cout << ans << endl;
}<commit_msg>submit D.cpp to 'D - Summer Vacation' (abc137) [C++14 (GCC 5.4.1)]<commit_after>#define DEBUG 1
/**
 * File    : D.cpp
 * Author  : Kazune Takahashi
 * Created : 8/11/2019, 12:04:39 AM
 * Powered by Visual Studio Code
 */
#include <iostream>
#include <iomanip>
#include <algorithm>
#include <vector>
#include <string>
#include <complex>
#include <tuple>
#include <queue>
#include <stack>
#include <map>
#include <set>
#include <unordered_map>
#include <unordered_set>
#include <bitset>
#include <functional>
#include <random>
#include <chrono>
#include <cctype>
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstdlib>
using namespace std;
#define maxs(x, y) (x = max(x, y))
#define mins(x, y) (x = min(x, y))
using ll = long long;
class mint
{
public:
  static ll MOD;
  ll x;
  mint() : x(0) {}
  mint(ll x) : x(x % MOD) {}
  mint operator-() const { return x ? MOD - x : 0; }
  mint &operator+=(const mint &a)
  {
    if ((x += a.x) >= MOD)
    {
      x -= MOD;
    }
    return *this;
  }
  mint &operator-=(const mint &a) { return *this += -a; }
  mint &operator*=(const mint &a)
  {
    (x *= a.x) %= MOD;
    return *this;
  }
  mint &operator/=(const mint &a)
  {
    mint b{a};
    return *this *= b.power(MOD - 2);
  }
  mint operator+(const mint &a) const { return mint(*this) += a; }
  mint operator-(const mint &a) const { return mint(*this) -= a; }
  mint operator*(const mint &a) const { return mint(*this) *= a; }
  mint operator/(const mint &a) const { return mint(*this) /= a; }
  bool operator<(const mint &a) const { return x < a.x; }
  bool operator==(const mint &a) const { return x == a.x; }
  const mint power(ll N)
  {
    if (N == 0)
    {
      return 1;
    }
    else if (N % 2 == 1)
    {
      return *this * power(N - 1);
    }
    else
    {
      mint half = power(N / 2);
      return half * half;
    }
  }
};
ll mint::MOD = 1e9 + 7;
istream &operator>>(istream &stream, mint &a) { return stream >> a.x; }
ostream &operator<<(ostream &stream, const mint &a) { return stream << a.x; }
class combination
{
public:
  vector<mint> inv, fact, factinv;
  static int MAX_SIZE;
  combination() : inv(MAX_SIZE), fact(MAX_SIZE), factinv(MAX_SIZE)
  {
    inv[1] = 1;
    for (auto i = 2; i < MAX_SIZE; i++)
    {
      inv[i] = (-inv[mint::MOD % i]) * (mint::MOD / i);
    }
    fact[0] = factinv[0] = 1;
    for (auto i = 1; i < MAX_SIZE; i++)
    {
      fact[i] = mint(i) * fact[i - 1];
      factinv[i] = inv[i] * factinv[i - 1];
    }
  }
  mint operator()(int n, int k)
  {
    if (n >= 0 && k >= 0 && n - k >= 0)
    {
      return fact[n] * factinv[k] * factinv[n - k];
    }
    return 0;
  }
};
int combination::MAX_SIZE = 3000010;
ll gcd(ll x, ll y) { return y ? gcd(y, x % y) : x; }
// constexpr double epsilon = 1e-10;
// constexpr ll infty = 1000000000000000LL;
// constexpr int dx[4] = {1, 0, -1, 0};
// constexpr int dy[4] = {0, 1, 0, -1};
void Yes()
{
  cout << "Yes" << endl;
  exit(0);
}
void No()
{
  cout << "No" << endl;
  exit(0);
}

int main()
{
  int N, M;
  cin >> N >> M;
  vector<vector<ll>> V(M, vector<ll>{});
  for (auto i = 0; i < N; i++)
  {
    int A;
    ll B;
    cin >> A >> B;
    int t = M - A;
    if (t >= 0)
    {
      V[t].push_back(B);
    }
  }
  priority_queue<ll> Q;
  ll ans = 0;
  for (auto i = M - 1; i >= 0; i--)
  {
    for (auto x : V[i])
    {
      Q.push(x);
    }
    if (!Q.empty())
    {
      ans += Q.top();
      Q.pop();
    }
  }
  cout << ans << endl;
}<|endoftext|>


<commit_before>#ifndef BUW_OPTIONALHIT_HPP
#define BUW_OPTIONALHIT_HPP

#include <iostream>
#include <memory>

#include "shape.hpp"

class Shape;

struct OptionalHit
{
	//default-constructor
	OptionalHit() :
	_hit{},
	_shape{},
	_t{0.0}
	{}

	OptionalHit(bool hit, std::shared_ptr<Shape> shape, double t) :
	_hit{hit},
	_shape{shape},
	_t{t}
	{}

	OptionalHit(bool hit, std::shared_ptr<Shape> shape, double t, glm::vec3 const& intersect, glm::vec3 const& normal) :
	_hit{hit},
	_shape{shape},
	_t{t},
	_intersect{intersect},
	_normal{normal}
	{}

	//variables
	bool _hit;
	int _depth;
	std::shared_ptr<Shape> _shape;
	double _t;
	glm::vec3 _intersect;
	glm::vec3 _normal;
};

#endif //#define BUW_OPTIONALHIT_HPP<commit_msg>added _ray as member variable<commit_after>#ifndef BUW_OPTIONALHIT_HPP
#define BUW_OPTIONALHIT_HPP

#include <iostream>
#include <memory>

#include "shape.hpp"

class Shape;

struct OptionalHit
{
	//default-constructor
	OptionalHit() :
	_hit{},
	_shape{},
	_depth{0},
	_t{0.0}
	{}

	OptionalHit(bool hit, std::shared_ptr<Shape> shape, double t) :
	_hit{hit},
	_depth{0},
	_shape{shape},
	_t{t}
	{}

	OptionalHit(bool hit, std::shared_ptr<Shape> shape, double t, glm::vec3 const& intersect, Ray const& ray, glm::vec3 const& normal) :
	_hit{hit},
	_depth{0},
	_shape{shape},
	_t{t},
	_intersect{intersect},
	_ray{ray},
	_normal{normal}
	{}

	//variables

	//wether the object was hit or not
	bool _hit;
	//recursion depth
	int _depth;
	//the object that was hit
	std::shared_ptr<Shape> _shape;
	//distance between _ray.origin and _intersect
	double _t;

	//intersecting point
	glm::vec3 _intersect;

	//intersecting ray
	Ray _ray;

	glm::vec3 _normal;
};

#endif //#define BUW_OPTIONALHIT_HPP<|endoftext|>


<commit_before>
// -*- mode: c++; c-basic-offset:4 -*-

// This file is part of ff_handler, a data handler for the OPeNDAP data
// server.

// Copyright (c) 2002,2003 OPeNDAP, Inc.
// Author: James Gallagher <jgallagher@opendap.org>
//
// This is free software; you can redistribute it and/or modify it under the
// terms of the GNU Lesser General Public License as published by the Free
// Software Foundation; either version 2.1 of the License, or (at your
// option) any later version.
//
// This software is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
// License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
//
// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.

// FFRequestHandler.cc

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>

#include "FFRequestHandler.h"

#include "config_ff.h"
#include "FFTypeFactory.h"

#include "DAS.h"
#include "DDS.h"
#include "BESInfo.h"
#include "BESResponseNames.h"
#include "BESContainer.h"
#include "BESResponseHandler.h"
#include "BESVersionInfo.h"
#include "BESDataNames.h"

long BufPtr = 0; // cache pointer
long BufSiz =0; // Cache size
char *BufVal = NULL; // cache buffer

extern void read_descriptors(DDS &das, const string &filename) throw(Error);
extern void get_attributes(DAS &das, string filename) throw(Error);

FFRequestHandler::FFRequestHandler( string name )
    : BESRequestHandler( name )
{
    add_handler( DAS_RESPONSE, FFRequestHandler::ff_build_das ) ;
    add_handler( DDS_RESPONSE, FFRequestHandler::ff_build_dds ) ;
    add_handler( DATA_RESPONSE, FFRequestHandler::ff_build_data ) ;
    add_handler( HELP_RESPONSE, FFRequestHandler::ff_build_help ) ;
    add_handler( VERS_RESPONSE, FFRequestHandler::ff_build_version ) ;
}

FFRequestHandler::~FFRequestHandler()
{
}

bool
FFRequestHandler::ff_build_das( BESDataHandlerInterface &dhi )
{
    DAS *das = (DAS *)dhi.response_handler->get_response_object() ;

    get_attributes( *das, dhi.container->get_real_name() ) ;

    return true ;
}

bool
FFRequestHandler::ff_build_dds( BESDataHandlerInterface &dhi )
{
    DDS *dds = (DDS *)dhi.response_handler->get_response_object() ;

    FFTypeFactory *factory = new FFTypeFactory ;
    dds->set_factory( factory ) ;

    read_descriptors( *dds, dhi.container->get_real_name() ) ;

    dhi.data[POST_CONSTRAINT] = dhi.container->get_constraint();

    dds->set_factory( NULL ) ;
    delete factory ;

    return true ;
}

bool
FFRequestHandler::ff_build_data( BESDataHandlerInterface &dhi )
{
    DDS *dds = (DDS *)dhi.response_handler->get_response_object() ;

    FFTypeFactory *factory = new FFTypeFactory ;
    dds->set_factory( factory ) ;

    // FIX: WHAT NOW???
    //register_functions(ce);
    dds->filename( dhi.container->get_real_name() ) ;
    read_descriptors( *dds, dhi.container->get_real_name() ) ;

    dhi.data[POST_CONSTRAINT] = dhi.container->get_constraint();

    dds->set_factory( NULL ) ;
    delete factory ;

    return true ;
}

bool
FFRequestHandler::ff_build_help( BESDataHandlerInterface &dhi )
{
    BESInfo *info = (BESInfo *)dhi.response_handler->get_response_object() ;
    string handles = (string)DAS_RESPONSE
                     + "," + DDS_RESPONSE
                     + "," + DATA_RESPONSE
                     + "," + HELP_RESPONSE
                     + "," + VERS_RESPONSE ;
    info->add_tag( "handles", handles ) ;
    info->add_tag( "version", PACKAGE_STRING ) ;

    return true ;
}

bool
FFRequestHandler::ff_build_version( BESDataHandlerInterface &dhi )
{
    BESVersionInfo *info = (BESVersionInfo *)dhi.response_handler->get_response_object() ;
    info->addHandlerVersion( PACKAGE_NAME, PACKAGE_VERSION ) ;
    return true ;
}

<commit_msg>In the help response added begin and end tags called handler to separate the different hander help responses and added a tag called name within the handler tag to say which handler this help is for<commit_after>
// -*- mode: c++; c-basic-offset:4 -*-

// This file is part of ff_handler, a data handler for the OPeNDAP data
// server.

// Copyright (c) 2002,2003 OPeNDAP, Inc.
// Author: James Gallagher <jgallagher@opendap.org>
//
// This is free software; you can redistribute it and/or modify it under the
// terms of the GNU Lesser General Public License as published by the Free
// Software Foundation; either version 2.1 of the License, or (at your
// option) any later version.
//
// This software is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
// License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
//
// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.

// FFRequestHandler.cc

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>

#include "FFRequestHandler.h"

#include "config_ff.h"
#include "FFTypeFactory.h"

#include "DAS.h"
#include "DDS.h"
#include "BESInfo.h"
#include "BESResponseNames.h"
#include "BESContainer.h"
#include "BESResponseHandler.h"
#include "BESVersionInfo.h"
#include "BESDataNames.h"

long BufPtr = 0; // cache pointer
long BufSiz =0; // Cache size
char *BufVal = NULL; // cache buffer

extern void read_descriptors(DDS &das, const string &filename) throw(Error);
extern void get_attributes(DAS &das, string filename) throw(Error);

FFRequestHandler::FFRequestHandler( string name )
    : BESRequestHandler( name )
{
    add_handler( DAS_RESPONSE, FFRequestHandler::ff_build_das ) ;
    add_handler( DDS_RESPONSE, FFRequestHandler::ff_build_dds ) ;
    add_handler( DATA_RESPONSE, FFRequestHandler::ff_build_data ) ;
    add_handler( HELP_RESPONSE, FFRequestHandler::ff_build_help ) ;
    add_handler( VERS_RESPONSE, FFRequestHandler::ff_build_version ) ;
}

FFRequestHandler::~FFRequestHandler()
{
}

bool
FFRequestHandler::ff_build_das( BESDataHandlerInterface &dhi )
{
    DAS *das = (DAS *)dhi.response_handler->get_response_object() ;

    get_attributes( *das, dhi.container->get_real_name() ) ;

    return true ;
}

bool
FFRequestHandler::ff_build_dds( BESDataHandlerInterface &dhi )
{
    DDS *dds = (DDS *)dhi.response_handler->get_response_object() ;

    FFTypeFactory *factory = new FFTypeFactory ;
    dds->set_factory( factory ) ;

    read_descriptors( *dds, dhi.container->get_real_name() ) ;

    dhi.data[POST_CONSTRAINT] = dhi.container->get_constraint();

    dds->set_factory( NULL ) ;
    delete factory ;

    return true ;
}

bool
FFRequestHandler::ff_build_data( BESDataHandlerInterface &dhi )
{
    DDS *dds = (DDS *)dhi.response_handler->get_response_object() ;

    FFTypeFactory *factory = new FFTypeFactory ;
    dds->set_factory( factory ) ;

    // FIX: WHAT NOW???
    //register_functions(ce);
    dds->filename( dhi.container->get_real_name() ) ;
    read_descriptors( *dds, dhi.container->get_real_name() ) ;

    dhi.data[POST_CONSTRAINT] = dhi.container->get_constraint();

    dds->set_factory( NULL ) ;
    delete factory ;

    return true ;
}

bool
FFRequestHandler::ff_build_help( BESDataHandlerInterface &dhi )
{
    BESInfo *info = (BESInfo *)dhi.response_handler->get_response_object() ;
    info->begin_tag( "Handler" ) ;
    info->add_tag( "name", PACKAGE_NAME ) ;
    string handles = (string)DAS_RESPONSE
                     + "," + DDS_RESPONSE
                     + "," + DATA_RESPONSE
                     + "," + HELP_RESPONSE
                     + "," + VERS_RESPONSE ;
    info->add_tag( "handles", handles ) ;
    info->add_tag( "version", PACKAGE_STRING ) ;
    info->end_tag( "Handler" ) ;

    return true ;
}

bool
FFRequestHandler::ff_build_version( BESDataHandlerInterface &dhi )
{
    BESVersionInfo *info = (BESVersionInfo *)dhi.response_handler->get_response_object() ;
    info->addHandlerVersion( PACKAGE_NAME, PACKAGE_VERSION ) ;
    return true ;
}

<|endoftext|>


<commit_before>#include <yuni/yuni.h>
#include <yuni/core/noncopyable.h>
#include <yuni/io/file.h>
#include <yuni/core/system/console.h>
#include <yuni/core/getopt.h>
#include <yuni/io/directory.h>
#include <yuni/io/directory/info.h>
#include <yuni/datetime/timestamp.h>
#include <yuni/core/logs/logs.h>
#include "nany/nany.h"
#include <algorithm>
#include <yuni/datetime/timestamp.h>
#include <iostream>
#include <vector>

using namespace Yuni;

namespace {

struct Settings {
	//! List of filenames to verify
	std::vector<String> filenames;
	// no colors
	bool noColors = false;
	// Result expected from filename convention
	bool useFilenameConvention = false;
};

template<class LeftType = Logs::NullDecorator>
struct ParseVerbosity : public LeftType {
	template<class Handler, class VerbosityType, class O>
	void internalDecoratorAddPrefix(O& out, const AnyString& s) const {
		// Write the verbosity to the output
		if (VerbosityType::hasName) {
			AnyString name{VerbosityType::Name()};
			if (s.empty()) {
			}
			else if (name == "info") {
				if (Handler::colorsAllowed)
					System::Console::TextColor<System::Console::yellow>::Set(out);
				#ifndef YUNI_OS_WINDOWS
				out << "      \u2713  ";
				#else
				out << "      >  ";
				#endif
				if (Handler::colorsAllowed)
					System::Console::TextColor<System::Console::white>::Set(out);
				out << "parsing";
				if (Handler::colorsAllowed)
					System::Console::ResetTextColor(out);
			}
			else if (name == "error") {
				if (Handler::colorsAllowed)
					System::Console::TextColor<System::Console::red>::Set(out);
				out << "  FAILED ";
				if (Handler::colorsAllowed)
					System::Console::TextColor<System::Console::white>::Set(out);
				out << "parsing";
				if (Handler::colorsAllowed)
					System::Console::ResetTextColor(out);
			}
			else if (name == "warning") {
				if (Handler::colorsAllowed)
					System::Console::TextColor<System::Console::yellow>::Set(out);
				out << "  {warn} ";
				if (Handler::colorsAllowed)
					System::Console::TextColor<System::Console::white>::Set(out);
				out << "parsing";
				if (Handler::colorsAllowed)
					System::Console::ResetTextColor(out);
			}
			else {
				// Set Color
				if (Handler::colorsAllowed && VerbosityType::color != System::Console::none)
					System::Console::TextColor<VerbosityType::color>::Set(out);
				// The verbosity
				VerbosityType::AppendName(out);
				// Reset Color
				if (Handler::colorsAllowed && VerbosityType::color != System::Console::none)
					System::Console::ResetTextColor(out);
			}
		}
		// Transmit the message to the next decorator
		LeftType::template internalDecoratorAddPrefix<Handler, VerbosityType,O>(out, s);
	}
}; // struct VerbosityLevel

using Logging = Logs::Logger<Logs::StdCout<>, ParseVerbosity<Logs::Message<>>>;
static Logging logs;

uint32_t fincCommonFolderLength(const std::vector<String>& filenames) {
	if (filenames.empty())
		return 0;
	auto& firstElement = filenames[0];
	const char sep = IO::Separator;
	uint32_t pos = 0;
	for (; ; ++pos) {
		for (size_t i = 0; i < filenames.size(); ++i) {
			auto& str = filenames[i];
			if (pos == firstElement.size())
				return pos;
			if (pos < str.size() and str[pos] != '\0' and str[pos] == firstElement[pos])
				continue;
			// back to the last sep
			while (pos > 0 && firstElement[--pos] != sep) {
			}
			return pos;
		}
	}
	return pos;
}

bool expandFilelist(std::vector<String>& list) {
	std::vector<String> filelist;
	filelist.reserve(512);
	String currentfile;
	currentfile.reserve(4096);
	for (auto& element: list) {
		IO::Canonicalize(currentfile, element);
		switch (IO::TypeOf(currentfile)) {
			case IO::typeFile: {
				filelist.emplace_back(currentfile);
				break;
			}
			case IO::typeFolder: {
				ShortString16 ext;
				IO::Directory::Info info(currentfile);
				auto end = info.recursive_file_end();
				for (auto i = info.recursive_file_begin(); i != end; ++i)
				{
					IO::ExtractExtension(ext, *i);
					if (ext == ".ny")
						filelist.emplace_back(i.filename());
				}
				break;
			}
			default: {
				logs.error() << "impossible to find '" << currentfile << "'";
				return false;
			}
		}
	}
	// for beauty in singled-threaded (and to always produce the same output)
	std::sort(filelist.begin(), filelist.end());
	list.swap(filelist);
	return true;
}

template<class F>
bool IterateThroughAllFiles(const std::vector<String>& filenames, const F& callback) {
	String currentfile;
	uint32_t testOK = 0;
	uint32_t testFAILED = 0;
	int64_t maxCheckDuration = 0;
	int64_t startTime = DateTime::NowMilliSeconds();
	for (auto& filename: filenames) {
		int64_t duration = 0;
		if (callback(filename, duration))
			++testOK;
		else
			++testFAILED;
		if (duration > maxCheckDuration)
			maxCheckDuration = duration;
	}
	int64_t endTime = DateTime::NowMilliSeconds();
	uint32_t total = testOK + testFAILED;
	if (total > 0) {
		int64_t duration = (endTime - startTime);
		String durationStr;
		durationStr << " (in " << duration << "ms, max: " << maxCheckDuration << "ms)";
		if (total > 1) {
			if (0 != testFAILED) {
				switch (total) {
					case 1:
						logs.warning() << "-- FAILED -- 1 file, +" << testOK << ", -" << testFAILED << durationStr;
						break;
					default:
						logs.warning() << "-- FAILED -- " << total << " files, +" << testOK << ", -" << testFAILED << durationStr;
				}
			}
			else
			{
				switch (total) {
					case 1:
						logs.info() << "success: 1 file, +" << testOK << ", -0" << durationStr;
						break;
					default:
						logs.info() << "success: " << total << " files, +" << testOK << ", -0" << durationStr;
				}
			}
		}
	}
	else
		logs.warning() << "no input file";
	return (0 == testFAILED);
}

bool batchCheckIfFilenamesConformToGrammar(Settings& settings) {
	if (not expandFilelist(settings.filenames))
		return false;
	auto commonFolder = (settings.filenames.size() > 1 ? fincCommonFolderLength(settings.filenames) : 0);
	if (0 != commonFolder)
		++commonFolder;
	return IterateThroughAllFiles(settings.filenames, [&](const AnyString& file, int64_t& duration) -> bool {
		String barefile;
		IO::ExtractFileName(barefile, file);
		bool expected = true;
		bool canfail = false;
		if (settings.useFilenameConvention)
		{
			if (barefile.startsWith("ko-"))
				expected = false;
			if (barefile.find("-canfail-") < barefile.size())
				canfail = true;
		}
		// PARSE
		int64_t start = DateTime::NowMilliSeconds();
		bool success = (nytrue == nytry_parse_file_n(file.c_str(), file.size()));
		duration = DateTime::NowMilliSeconds() - start;
		success = (success == expected);
		if (success and duration < 300) {
			logs.info() << AnyString{file, commonFolder} << " [" << duration << "ms]";
		}
		else {
			if (not success) {
				if (not canfail)
					logs.error() << AnyString{file, commonFolder} << " [" << duration << "ms]";
				else
					logs.warning() << AnyString{file, commonFolder} << " [" << duration << "ms, can fail]";
			}
			else
				logs.error() << AnyString{file, commonFolder} << " [" << duration << "ms - time limit reached]";
			success = canfail;
		}
		return success;
	});
}

} // namespace

int main(int argc, char** argv)
{
	Settings settings;
	// parse the command
	{
		// The command line options parser
		GetOpt::Parser options;
		// Input files
		options.add(settings.filenames, 'i', "input", "Input files (or folders)");
		options.remainingArguments(settings.filenames);
		// --no-color
		options.addFlag(settings.noColors, ' ', "no-color", "Disable color output");
		// use filename convention
		options.addFlag(settings.useFilenameConvention, ' ', "use-filename-convention",
			"Use the filename to determine if the test should succeed or not (should succeed if starting with 'ok-'");
		// version
		bool optVersion = false;
		options.addFlag(optVersion, ' ', "version", "Display the version of the compiler and exit");
		// Ask to the parser to parse the command line
		if (not options(argc, argv)) {
			// The program should not continue here
			// The user may have requested the help or an error has happened
			// If an error has happened, the exit status should be different from 0
			if (options.errors()) {
				std::cerr << "Abort due to error\n";
				return EXIT_FAILURE;
			}
			return EXIT_SUCCESS;
		}

		if (optVersion) {
			std::cout << "0.0\n";
			return EXIT_SUCCESS;
		}
		if (settings.filenames.empty()) {
			std::cerr << argv[0] << ": no input file\n";
			return EXIT_FAILURE;
		}
	}
	// Print AST or check for Nany Grammar
	bool success = batchCheckIfFilenamesConformToGrammar(settings);
	return success ? EXIT_SUCCESS : EXIT_FAILURE;
}
<commit_msg>check-syntax: simplified function for retrieving the common subfolder<commit_after>#include <yuni/yuni.h>
#include <yuni/core/noncopyable.h>
#include <yuni/io/file.h>
#include <yuni/core/system/console.h>
#include <yuni/core/getopt.h>
#include <yuni/io/directory.h>
#include <yuni/io/directory/info.h>
#include <yuni/datetime/timestamp.h>
#include <yuni/core/logs/logs.h>
#include "nany/nany.h"
#include <algorithm>
#include <yuni/datetime/timestamp.h>
#include <iostream>
#include <vector>

using namespace Yuni;

namespace {

struct Settings {
	//! List of filenames to verify
	std::vector<String> filenames;
	// no colors
	bool noColors = false;
	// Result expected from filename convention
	bool useFilenameConvention = false;
};

template<class LeftType = Logs::NullDecorator>
struct ParseVerbosity : public LeftType {
	template<class Handler, class VerbosityType, class O>
	void internalDecoratorAddPrefix(O& out, const AnyString& s) const {
		// Write the verbosity to the output
		if (VerbosityType::hasName) {
			AnyString name{VerbosityType::Name()};
			if (s.empty()) {
			}
			else if (name == "info") {
				if (Handler::colorsAllowed)
					System::Console::TextColor<System::Console::yellow>::Set(out);
				#ifndef YUNI_OS_WINDOWS
				out << "      \u2713  ";
				#else
				out << "      >  ";
				#endif
				if (Handler::colorsAllowed)
					System::Console::TextColor<System::Console::white>::Set(out);
				out << "parsing";
				if (Handler::colorsAllowed)
					System::Console::ResetTextColor(out);
			}
			else if (name == "error") {
				if (Handler::colorsAllowed)
					System::Console::TextColor<System::Console::red>::Set(out);
				out << "  FAILED ";
				if (Handler::colorsAllowed)
					System::Console::TextColor<System::Console::white>::Set(out);
				out << "parsing";
				if (Handler::colorsAllowed)
					System::Console::ResetTextColor(out);
			}
			else if (name == "warning") {
				if (Handler::colorsAllowed)
					System::Console::TextColor<System::Console::yellow>::Set(out);
				out << "  {warn} ";
				if (Handler::colorsAllowed)
					System::Console::TextColor<System::Console::white>::Set(out);
				out << "parsing";
				if (Handler::colorsAllowed)
					System::Console::ResetTextColor(out);
			}
			else {
				// Set Color
				if (Handler::colorsAllowed && VerbosityType::color != System::Console::none)
					System::Console::TextColor<VerbosityType::color>::Set(out);
				// The verbosity
				VerbosityType::AppendName(out);
				// Reset Color
				if (Handler::colorsAllowed && VerbosityType::color != System::Console::none)
					System::Console::ResetTextColor(out);
			}
		}
		// Transmit the message to the next decorator
		LeftType::template internalDecoratorAddPrefix<Handler, VerbosityType,O>(out, s);
	}
}; // struct VerbosityLevel

using Logging = Logs::Logger<Logs::StdCout<>, ParseVerbosity<Logs::Message<>>>;
static Logging logs;

uint32_t fincCommonFolderLength(const std::vector<String>& filenames) {
	if (filenames.empty())
		return 0;
	uint32_t len = 0;
	auto& firstElement = filenames[0];
	const char sep = IO::Separator;
	for (; ; ++len) {
		for (auto& filename: filenames) {
			if (len == firstElement.size())
				return len;
			if (len < filename.size() and filename[len] != '\0' and filename[len] == firstElement[len])
				continue;
			while (len > 0 and firstElement[--len] != sep) { // back to the last sep
			}
			return len;
		}
	}
	return len;
}

bool expandFilelist(std::vector<String>& list) {
	std::vector<String> filelist;
	filelist.reserve(512);
	String currentfile;
	currentfile.reserve(4096);
	for (auto& element: list) {
		IO::Canonicalize(currentfile, element);
		switch (IO::TypeOf(currentfile)) {
			case IO::typeFile: {
				filelist.emplace_back(currentfile);
				break;
			}
			case IO::typeFolder: {
				ShortString16 ext;
				IO::Directory::Info info(currentfile);
				auto end = info.recursive_file_end();
				for (auto i = info.recursive_file_begin(); i != end; ++i)
				{
					IO::ExtractExtension(ext, *i);
					if (ext == ".ny")
						filelist.emplace_back(i.filename());
				}
				break;
			}
			default: {
				logs.error() << "impossible to find '" << currentfile << "'";
				return false;
			}
		}
	}
	// for beauty in singled-threaded (and to always produce the same output)
	std::sort(filelist.begin(), filelist.end());
	list.swap(filelist);
	return true;
}

template<class F>
bool IterateThroughAllFiles(const std::vector<String>& filenames, const F& callback) {
	String currentfile;
	uint32_t testOK = 0;
	uint32_t testFAILED = 0;
	int64_t maxCheckDuration = 0;
	int64_t startTime = DateTime::NowMilliSeconds();
	for (auto& filename: filenames) {
		int64_t duration = 0;
		if (callback(filename, duration))
			++testOK;
		else
			++testFAILED;
		if (duration > maxCheckDuration)
			maxCheckDuration = duration;
	}
	int64_t endTime = DateTime::NowMilliSeconds();
	uint32_t total = testOK + testFAILED;
	if (total > 0) {
		int64_t duration = (endTime - startTime);
		String durationStr;
		durationStr << " (in " << duration << "ms, max: " << maxCheckDuration << "ms)";
		if (total > 1) {
			if (0 != testFAILED) {
				switch (total) {
					case 1:
						logs.warning() << "-- FAILED -- 1 file, +" << testOK << ", -" << testFAILED << durationStr;
						break;
					default:
						logs.warning() << "-- FAILED -- " << total << " files, +" << testOK << ", -" << testFAILED << durationStr;
				}
			}
			else
			{
				switch (total) {
					case 1:
						logs.info() << "success: 1 file, +" << testOK << ", -0" << durationStr;
						break;
					default:
						logs.info() << "success: " << total << " files, +" << testOK << ", -0" << durationStr;
				}
			}
		}
	}
	else
		logs.warning() << "no input file";
	return (0 == testFAILED);
}

bool batchCheckIfFilenamesConformToGrammar(Settings& settings) {
	if (not expandFilelist(settings.filenames))
		return false;
	auto commonFolder = (settings.filenames.size() > 1 ? fincCommonFolderLength(settings.filenames) : 0);
	if (0 != commonFolder)
		++commonFolder;
	return IterateThroughAllFiles(settings.filenames, [&](const AnyString& file, int64_t& duration) -> bool {
		String barefile;
		IO::ExtractFileName(barefile, file);
		bool expected = true;
		bool canfail = false;
		if (settings.useFilenameConvention)
		{
			if (barefile.startsWith("ko-"))
				expected = false;
			if (barefile.find("-canfail-") < barefile.size())
				canfail = true;
		}
		// PARSE
		int64_t start = DateTime::NowMilliSeconds();
		bool success = (nytrue == nytry_parse_file_n(file.c_str(), file.size()));
		duration = DateTime::NowMilliSeconds() - start;
		success = (success == expected);
		if (success and duration < 300) {
			logs.info() << AnyString{file, commonFolder} << " [" << duration << "ms]";
		}
		else {
			if (not success) {
				if (not canfail)
					logs.error() << AnyString{file, commonFolder} << " [" << duration << "ms]";
				else
					logs.warning() << AnyString{file, commonFolder} << " [" << duration << "ms, can fail]";
			}
			else
				logs.error() << AnyString{file, commonFolder} << " [" << duration << "ms - time limit reached]";
			success = canfail;
		}
		return success;
	});
}

} // namespace

int main(int argc, char** argv)
{
	Settings settings;
	// parse the command
	{
		// The command line options parser
		GetOpt::Parser options;
		// Input files
		options.add(settings.filenames, 'i', "input", "Input files (or folders)");
		options.remainingArguments(settings.filenames);
		// --no-color
		options.addFlag(settings.noColors, ' ', "no-color", "Disable color output");
		// use filename convention
		options.addFlag(settings.useFilenameConvention, ' ', "use-filename-convention",
			"Use the filename to determine if the test should succeed or not (should succeed if starting with 'ok-'");
		// version
		bool optVersion = false;
		options.addFlag(optVersion, ' ', "version", "Display the version of the compiler and exit");
		// Ask to the parser to parse the command line
		if (not options(argc, argv)) {
			// The program should not continue here
			// The user may have requested the help or an error has happened
			// If an error has happened, the exit status should be different from 0
			if (options.errors()) {
				std::cerr << "Abort due to error\n";
				return EXIT_FAILURE;
			}
			return EXIT_SUCCESS;
		}

		if (optVersion) {
			std::cout << "0.0\n";
			return EXIT_SUCCESS;
		}
		if (settings.filenames.empty()) {
			std::cerr << argv[0] << ": no input file\n";
			return EXIT_FAILURE;
		}
	}
	// Print AST or check for Nany Grammar
	bool success = batchCheckIfFilenamesConformToGrammar(settings);
	return success ? EXIT_SUCCESS : EXIT_FAILURE;
}
<|endoftext|>


<commit_before>#include <bit/memory/aligned_memory.hpp>

#include <unistd.h>

#include <cstdlib>
#include <cstdint>

void* bit::memory::aligned_malloc( std::size_t size, std::size_t align )
  noexcept
{
  void* result_ptr;
  ::posix_memalign(&result_ptr, align, size);
  return result_ptr;
}

//------------------------------------------------------------------------

void bit::memory::aligned_free( void* ptr )
{
  std::free(ptr);
}

//------------------------------------------------------------------------

void* bit::memory::aligned_offset_malloc( std::size_t size,
                                          std::size_t align,
                                          std::size_t offset )
  noexcept
{
  union {
    void*         aligned_ptr;
    std::uint16_t adjust_ptr;
  } r = {};

  void* allocated_ptr;

  // Add the amount adjusted for the offset here.
  // 16 bits chosen in case an offset greater than 255 is necessary (unlikely)
  offset += sizeof(std::uint16_t);

  // Step offset up to a multiple of the alignment
  size += offset + (offset % align);

  ::posix_memalign(&allocated_ptr, align, size);

  // Find the amount adjusted and store th result
  std::size_t adjust;
  //r.aligned_ptr = Pointer_Math::align_forward( allocated_ptr, align, offset, &adjust );

  *(r.adjust_ptr++) = static_cast<std::uint16_t>(adjust);

  return r.aligned_ptr;
}

//------------------------------------------------------------------------

void bit::memory::aligned_offset_free( void* ptr )
{
  union{
    void*          aligned_ptr;
    std::uint16_t* adjust_ptr;
  } r = {};
  r.aligned_ptr = ptr;
  std::uint16_t adjust = *(--r.adjust_ptr);

  //  r.aligned_ptr = Pointer_Math::subtract(r.aligned_ptr, adjust);

  std::free( ptr );
}

#define BIT_ALIGNED_MALLOC_DEFINED
<commit_msg>Fixed broke aligned offset allocation<commit_after>#include <bit/memory/aligned_memory.hpp>

#include <unistd.h>

#include <cstdlib>
#include <cstdint>

void* bit::memory::aligned_malloc( std::size_t size, std::size_t align )
  noexcept
{
  void* result_ptr;
  ::posix_memalign(&result_ptr, align, size);
  return result_ptr;
}

//------------------------------------------------------------------------

void bit::memory::aligned_free( void* ptr )
{
  std::free(ptr);
}

//------------------------------------------------------------------------

void* bit::memory::aligned_offset_malloc( std::size_t size,
                                          std::size_t align,
                                          std::size_t offset )
  noexcept
{
  union {
    void*          aligned_ptr;
    std::uint16_t* adjust_ptr;
  } r = {};

  assert( align % 2 == 0 && "Alignment must be power of 2" );

  void* allocated_ptr;

  // Add the amount adjusted for the offset here.
  // 16 bits chosen in case an offset greater than 255 is necessary (unlikely)
  offset += sizeof(std::uint16_t);

  // Step offset up to a multiple of the alignment
  size += offset + (offset & align);

  ::posix_memalign(&allocated_ptr, align, size);

  // Find the amount adjusted and store th result
  std::size_t adjust;
  //r.aligned_ptr = Pointer_Math::align_forward( allocated_ptr, align, offset, &adjust );

  *(r.adjust_ptr++) = static_cast<std::uint16_t>(adjust);

  return r.aligned_ptr;
}

//------------------------------------------------------------------------

void bit::memory::aligned_offset_free( void* ptr )
{
  union{
    void*          aligned_ptr;
    std::uint16_t* adjust_ptr;
  } r = {};
  r.aligned_ptr = ptr;
  std::uint16_t adjust = *(--r.adjust_ptr);

  auto p = static_cast<char*>(r.adjust_ptr) - adjust;
  //  r.aligned_ptr = Pointer_Math::subtract(r.aligned_ptr, adjust);

  std::free( ptr );
}

#define BIT_ALIGNED_MALLOC_DEFINED
<|endoftext|>


<commit_before>#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <unordered_map>

#include "cbor.h"

static size_t allocated_mem = 0;
static std::unordered_map<void*, size_t> allocated_len_map;

void *limited_malloc(size_t size) {
    if (size + allocated_mem > 1 << 30) {
        return nullptr;
    }
    void* m = malloc(size);
    if (m != nullptr) {
      allocated_mem += size;
      allocated_len_map[m] = size;
    }
    return m;
}

void limited_free(void *ptr) {
  if (allocated_len_map.find(ptr) == allocated_len_map.end()) {
    abort();
  }
  free(ptr);
  allocated_mem -= allocated_len_map[ptr];
  allocated_len_map.erase(ptr);
}

void *limited_realloc(void *ptr, size_t size) {
  limited_free(ptr);
  return limited_malloc(size);
}

struct State {
    FILE* fout;

    State() : fout(fopen("/dev/null", "r")) {
        cbor_set_allocs(limited_malloc, limited_realloc, limited_free);
    }
};

static State kState;

extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
    cbor_load_result result;
    cbor_item_t *item = cbor_load(Data, Size, &result);
    if (result.error.code == CBOR_ERR_NONE) {
        cbor_describe(item, kState.fout);
        unsigned char *buffer;
        size_t buffer_size;
        cbor_serialize_alloc(item, &buffer, &buffer_size);
        free(buffer);
        cbor_item_t *copied = cbor_copy(item);
        cbor_decref(&copied);
        cbor_decref(&item);
    }
    return 0;
}
<commit_msg>Fix fuzzer realloc<commit_after>#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <unordered_map>

#include "cbor.h"

static size_t allocated_mem = 0;
static std::unordered_map<void*, size_t> allocated_len_map;
static constexpr size_t kMemoryLimit = 1 << 30;

void *limited_malloc(size_t size) {
    if (size + allocated_mem > kMemoryLimit) {
        return nullptr;
    }
    void* m = malloc(size);
    if (m != nullptr) {
      allocated_mem += size;
      allocated_len_map[m] = size;
    }
    return m;
}

void limited_free(void *ptr) {
  if (allocated_len_map.find(ptr) == allocated_len_map.end()) {
    abort();
  }
  free(ptr);
  allocated_mem -= allocated_len_map[ptr];
  allocated_len_map.erase(ptr);
}

void *limited_realloc(void *ptr, size_t size) {
  if (allocated_len_map.find(ptr) == allocated_len_map.end()) {
    abort();
  }
  long delta = (long) size - allocated_len_map[ptr];
  if (delta + allocated_mem > kMemoryLimit) {
    return nullptr;
  }
  void* new_ptr = realloc(ptr, size);
  if (new_ptr == nullptr) {
    return nullptr;
  }
  allocated_mem += delta;
  allocated_len_map.erase(ptr);
  allocated_len_map[new_ptr] = size;

  return new_ptr;
}

struct State {
    FILE* fout;

    State() : fout(fopen("/dev/null", "r")) {
        cbor_set_allocs(limited_malloc, limited_realloc, limited_free);
    }
};

static State kState;

extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
    cbor_load_result result;
    cbor_item_t *item = cbor_load(Data, Size, &result);
    if (result.error.code == CBOR_ERR_NONE) {
        cbor_describe(item, kState.fout);
        unsigned char *buffer;
        size_t buffer_size;
        cbor_serialize_alloc(item, &buffer, &buffer_size);
        free(buffer);
        cbor_item_t *copied = cbor_copy(item);
        cbor_decref(&copied);
        cbor_decref(&item);
    }
    return 0;
}
<|endoftext|>


<commit_before>/*
 *
 *    Copyright (c) 2020-2021 Project CHIP Authors
 *    All rights reserved.
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

/**
 *  @file
 *    This file contains implementation of Device class. The objects of this
 *    class will be used by Controller applications to interact with CHIP
 *    devices. The class provides mechanism to construct, send and receive
 *    messages to and from the corresponding CHIP devices.
 */

#include <controller/CommissioneeDeviceProxy.h>

#include <controller-clusters/zap-generated/CHIPClusters.h>

#include <app/CommandSender.h>
#include <app/ReadPrepareParams.h>
#include <app/util/DataModelHandler.h>
#include <lib/core/CHIPCore.h>
#include <lib/core/CHIPEncoding.h>
#include <lib/core/CHIPSafeCasts.h>
#include <lib/support/CodeUtils.h>
#include <lib/support/ErrorStr.h>
#include <lib/support/SafeInt.h>
#include <lib/support/logging/CHIPLogging.h>

using namespace chip::Callback;

namespace chip {

// TODO - Refactor LoadSecureSessionParametersIfNeeded() as device object is no longer persisted
CHIP_ERROR CommissioneeDeviceProxy::LoadSecureSessionParametersIfNeeded(bool & didLoad)
{
    didLoad = false;

    // If there is no secure connection to the device, try establishing it
    if (mState != ConnectionState::SecureConnected)
    {
        ReturnErrorOnFailure(LoadSecureSessionParameters());
        didLoad = true;
    }
    else
    {
        if (mSecureSession)
        {
            // Check if the connection state has the correct transport information
            if (mSecureSession->AsSecureSession()->GetPeerAddress().GetTransportType() == Transport::Type::kUndefined)
            {
                mState = ConnectionState::NotConnected;
                ReturnErrorOnFailure(LoadSecureSessionParameters());
                didLoad = true;
            }
        }
        else
        {
            mState = ConnectionState::NotConnected;
            ReturnErrorOnFailure(LoadSecureSessionParameters());
            didLoad = true;
        }
    }

    return CHIP_NO_ERROR;
}

CHIP_ERROR CommissioneeDeviceProxy::SendCommands(app::CommandSender * commandObj)
{
    bool loadedSecureSession = false;
    ReturnErrorOnFailure(LoadSecureSessionParametersIfNeeded(loadedSecureSession));
    VerifyOrReturnError(commandObj != nullptr, CHIP_ERROR_INVALID_ARGUMENT);
    return commandObj->SendCommandRequest(mSecureSession.Get());
}

void CommissioneeDeviceProxy::OnSessionReleased()
{
    mState = ConnectionState::NotConnected;
}

CHIP_ERROR CommissioneeDeviceProxy::CloseSession()
{
    ReturnErrorCodeIf(mState != ConnectionState::SecureConnected, CHIP_ERROR_INCORRECT_STATE);
    if (mSecureSession)
    {
        mSessionManager->ExpirePairing(mSecureSession.Get());
    }
    mState = ConnectionState::NotConnected;
    mPairing.Clear();
    return CHIP_NO_ERROR;
}

CHIP_ERROR CommissioneeDeviceProxy::UpdateDeviceData(const Transport::PeerAddress & addr,
                                                     const ReliableMessageProtocolConfig & config)
{
    bool didLoad;

    mDeviceAddress = addr;

    mMRPConfig = config;

    // Initialize PASE session state with any MRP parameters that DNS-SD has provided.
    // It can be overridden by PASE session protocol messages that include MRP parameters.
    mPairing.SetMRPConfig(mMRPConfig);

    ReturnErrorOnFailure(LoadSecureSessionParametersIfNeeded(didLoad));

    if (!mSecureSession)
    {
        // Nothing needs to be done here.  It's not an error to not have a
        // secureSession.  For one thing, we could have gotten an different
        // UpdateAddress already and that caused connections to be torn down and
        // whatnot.
        return CHIP_NO_ERROR;
    }

    Transport::SecureSession * secureSession = mSecureSession.Get()->AsSecureSession();
    secureSession->SetPeerAddress(addr);

    return CHIP_NO_ERROR;
}

CHIP_ERROR CommissioneeDeviceProxy::SetConnected()
{
    if (mState != ConnectionState::Connecting)
    {
        return CHIP_ERROR_INCORRECT_STATE;
    }
    bool _didLoad;
    CHIP_ERROR err = LoadSecureSessionParametersIfNeeded(_didLoad);
    if (err == CHIP_NO_ERROR)
    {
        mState = ConnectionState::SecureConnected;
    }
    return err;
}

void CommissioneeDeviceProxy::Reset()
{
    SetActive(false);

    mState              = ConnectionState::NotConnected;
    mSessionManager     = nullptr;
    mUDPEndPointManager = nullptr;
#if CONFIG_NETWORK_LAYER_BLE
    mBleLayer = nullptr;
#endif
    mExchangeMgr = nullptr;

    ReleaseDAC();
    ReleasePAI();
}

CHIP_ERROR CommissioneeDeviceProxy::LoadSecureSessionParameters()
{
    CHIP_ERROR err = CHIP_NO_ERROR;
    SessionHolder sessionHolder;

    if (mSessionManager == nullptr || mState == ConnectionState::SecureConnected)
    {
        ExitNow(err = CHIP_ERROR_INCORRECT_STATE);
    }

    if (mState == ConnectionState::Connecting)
    {
        ExitNow(err = CHIP_NO_ERROR);
    }

    SuccessOrExit(mSessionManager->NewPairing(mSecureSession, Optional<Transport::PeerAddress>::Value(mDeviceAddress), mDeviceId,
                                              &mPairing, CryptoContext::SessionRole::kInitiator, mFabricIndex));
    mState = ConnectionState::SecureConnected;

exit:

    if (err != CHIP_NO_ERROR)
    {
        ChipLogError(Controller, "LoadSecureSessionParameters returning error %" CHIP_ERROR_FORMAT, err.Format());
    }
    return err;
}

bool CommissioneeDeviceProxy::GetAddress(Inet::IPAddress & addr, uint16_t & port) const
{
    if (mState == ConnectionState::NotConnected)
        return false;

    addr = mDeviceAddress.GetIPAddress();
    port = mDeviceAddress.GetPort();
    return true;
}

void CommissioneeDeviceProxy::ReleaseDAC()
{
    if (mDAC != nullptr)
    {
        Platform::MemoryFree(mDAC);
    }
    mDACLen = 0;
    mDAC    = nullptr;
}

CHIP_ERROR CommissioneeDeviceProxy::SetDAC(const ByteSpan & dac)
{
    if (dac.size() == 0)
    {
        ReleaseDAC();
        return CHIP_NO_ERROR;
    }

    VerifyOrReturnError(dac.size() <= Credentials::kMaxDERCertLength, CHIP_ERROR_INVALID_ARGUMENT);
    if (mDACLen != 0)
    {
        ReleaseDAC();
    }

    VerifyOrReturnError(CanCastTo<uint16_t>(dac.size()), CHIP_ERROR_INVALID_ARGUMENT);
    if (mDAC == nullptr)
    {
        mDAC = static_cast<uint8_t *>(chip::Platform::MemoryAlloc(dac.size()));
    }
    VerifyOrReturnError(mDAC != nullptr, CHIP_ERROR_NO_MEMORY);
    mDACLen = static_cast<uint16_t>(dac.size());
    memcpy(mDAC, dac.data(), mDACLen);

    return CHIP_NO_ERROR;
}

void CommissioneeDeviceProxy::ReleasePAI()
{
    if (mPAI != nullptr)
    {
        chip::Platform::MemoryFree(mPAI);
    }
    mPAILen = 0;
    mPAI    = nullptr;
}

CHIP_ERROR CommissioneeDeviceProxy::SetPAI(const chip::ByteSpan & pai)
{
    if (pai.size() == 0)
    {
        ReleasePAI();
        return CHIP_NO_ERROR;
    }

    VerifyOrReturnError(pai.size() <= Credentials::kMaxDERCertLength, CHIP_ERROR_INVALID_ARGUMENT);
    if (mPAILen != 0)
    {
        ReleasePAI();
    }

    VerifyOrReturnError(CanCastTo<uint16_t>(pai.size()), CHIP_ERROR_INVALID_ARGUMENT);
    if (mPAI == nullptr)
    {
        mPAI = static_cast<uint8_t *>(chip::Platform::MemoryAlloc(pai.size()));
    }
    VerifyOrReturnError(mPAI != nullptr, CHIP_ERROR_NO_MEMORY);
    mPAILen = static_cast<uint16_t>(pai.size());
    memcpy(mPAI, pai.data(), mPAILen);

    return CHIP_NO_ERROR;
}

CommissioneeDeviceProxy::~CommissioneeDeviceProxy()
{
    ReleaseDAC();
    ReleasePAI();
}

CHIP_ERROR CommissioneeDeviceProxy::SetNOCCertBufferSize(size_t new_size)
{
    ReturnErrorCodeIf(new_size > sizeof(mNOCCertBuffer), CHIP_ERROR_INVALID_ARGUMENT);
    mNOCCertBufferSize = new_size;
    return CHIP_NO_ERROR;
}

CHIP_ERROR CommissioneeDeviceProxy::SetICACertBufferSize(size_t new_size)
{
    ReturnErrorCodeIf(new_size > sizeof(mICACertBuffer), CHIP_ERROR_INVALID_ARGUMENT);
    mICACertBufferSize = new_size;
    return CHIP_NO_ERROR;
}

} // namespace chip
<commit_msg>CommissioneeDeviceProxy fix. (#13482)<commit_after>/*
 *
 *    Copyright (c) 2020-2021 Project CHIP Authors
 *    All rights reserved.
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

/**
 *  @file
 *    This file contains implementation of Device class. The objects of this
 *    class will be used by Controller applications to interact with CHIP
 *    devices. The class provides mechanism to construct, send and receive
 *    messages to and from the corresponding CHIP devices.
 */

#include <controller/CommissioneeDeviceProxy.h>

#include <controller-clusters/zap-generated/CHIPClusters.h>

#include <app/CommandSender.h>
#include <app/ReadPrepareParams.h>
#include <app/util/DataModelHandler.h>
#include <lib/core/CHIPCore.h>
#include <lib/core/CHIPEncoding.h>
#include <lib/core/CHIPSafeCasts.h>
#include <lib/support/CodeUtils.h>
#include <lib/support/ErrorStr.h>
#include <lib/support/SafeInt.h>
#include <lib/support/logging/CHIPLogging.h>

using namespace chip::Callback;

namespace chip {

// TODO - Refactor LoadSecureSessionParametersIfNeeded() as device object is no longer persisted
CHIP_ERROR CommissioneeDeviceProxy::LoadSecureSessionParametersIfNeeded(bool & didLoad)
{
    didLoad = false;

    // If there is no secure connection to the device, try establishing it
    if (mState != ConnectionState::SecureConnected)
    {
        ReturnErrorOnFailure(LoadSecureSessionParameters());
        didLoad = true;
    }
    else
    {
        if (mSecureSession)
        {
            // Check if the connection state has the correct transport information
            if (mSecureSession->AsSecureSession()->GetPeerAddress().GetTransportType() == Transport::Type::kUndefined)
            {
                mState = ConnectionState::NotConnected;
                ReturnErrorOnFailure(LoadSecureSessionParameters());
                didLoad = true;
            }
        }
        else
        {
            mState = ConnectionState::NotConnected;
            ReturnErrorOnFailure(LoadSecureSessionParameters());
            didLoad = true;
        }
    }

    return CHIP_NO_ERROR;
}

CHIP_ERROR CommissioneeDeviceProxy::SendCommands(app::CommandSender * commandObj)
{
    bool loadedSecureSession = false;
    ReturnErrorOnFailure(LoadSecureSessionParametersIfNeeded(loadedSecureSession));
    VerifyOrReturnError(commandObj != nullptr, CHIP_ERROR_INVALID_ARGUMENT);
    return commandObj->SendCommandRequest(mSecureSession.Get());
}

void CommissioneeDeviceProxy::OnSessionReleased()
{
    mState = ConnectionState::NotConnected;
}

CHIP_ERROR CommissioneeDeviceProxy::CloseSession()
{
    ReturnErrorCodeIf(mState != ConnectionState::SecureConnected, CHIP_ERROR_INCORRECT_STATE);
    if (mSecureSession)
    {
        mSessionManager->ExpirePairing(mSecureSession.Get());
    }
    mState = ConnectionState::NotConnected;
    mPairing.Clear();
    return CHIP_NO_ERROR;
}

CHIP_ERROR CommissioneeDeviceProxy::UpdateDeviceData(const Transport::PeerAddress & addr,
                                                     const ReliableMessageProtocolConfig & config)
{
    bool didLoad;

    mDeviceAddress = addr;

    mMRPConfig = config;

    // Initialize PASE session state with any MRP parameters that DNS-SD has provided.
    // It can be overridden by PASE session protocol messages that include MRP parameters.
    mPairing.SetMRPConfig(mMRPConfig);

    ReturnErrorOnFailure(LoadSecureSessionParametersIfNeeded(didLoad));

    if (!mSecureSession)
    {
        // Nothing needs to be done here.  It's not an error to not have a
        // secureSession.  For one thing, we could have gotten an different
        // UpdateAddress already and that caused connections to be torn down and
        // whatnot.
        return CHIP_NO_ERROR;
    }

    Transport::SecureSession * secureSession = mSecureSession.Get()->AsSecureSession();
    secureSession->SetPeerAddress(addr);

    return CHIP_NO_ERROR;
}

CHIP_ERROR CommissioneeDeviceProxy::SetConnected()
{
    if (mState != ConnectionState::Connecting)
    {
        return CHIP_ERROR_INCORRECT_STATE;
    }
    mState = ConnectionState::SecureConnected;
    bool _didLoad;
    CHIP_ERROR err = LoadSecureSessionParametersIfNeeded(_didLoad);
    if (err != CHIP_NO_ERROR)
    {
        mState = ConnectionState::NotConnected;
    }
    return err;
}

void CommissioneeDeviceProxy::Reset()
{
    SetActive(false);

    mState              = ConnectionState::NotConnected;
    mSessionManager     = nullptr;
    mUDPEndPointManager = nullptr;
#if CONFIG_NETWORK_LAYER_BLE
    mBleLayer = nullptr;
#endif
    mExchangeMgr = nullptr;

    ReleaseDAC();
    ReleasePAI();
}

CHIP_ERROR CommissioneeDeviceProxy::LoadSecureSessionParameters()
{
    CHIP_ERROR err = CHIP_NO_ERROR;
    SessionHolder sessionHolder;

    if (mSessionManager == nullptr || mState == ConnectionState::SecureConnected)
    {
        ExitNow(err = CHIP_ERROR_INCORRECT_STATE);
    }

    if (mState == ConnectionState::Connecting)
    {
        ExitNow(err = CHIP_NO_ERROR);
    }

    SuccessOrExit(mSessionManager->NewPairing(mSecureSession, Optional<Transport::PeerAddress>::Value(mDeviceAddress), mDeviceId,
                                              &mPairing, CryptoContext::SessionRole::kInitiator, mFabricIndex));
    mState = ConnectionState::SecureConnected;

exit:

    if (err != CHIP_NO_ERROR)
    {
        ChipLogError(Controller, "LoadSecureSessionParameters returning error %" CHIP_ERROR_FORMAT, err.Format());
    }
    return err;
}

bool CommissioneeDeviceProxy::GetAddress(Inet::IPAddress & addr, uint16_t & port) const
{
    if (mState == ConnectionState::NotConnected)
        return false;

    addr = mDeviceAddress.GetIPAddress();
    port = mDeviceAddress.GetPort();
    return true;
}

void CommissioneeDeviceProxy::ReleaseDAC()
{
    if (mDAC != nullptr)
    {
        Platform::MemoryFree(mDAC);
    }
    mDACLen = 0;
    mDAC    = nullptr;
}

CHIP_ERROR CommissioneeDeviceProxy::SetDAC(const ByteSpan & dac)
{
    if (dac.size() == 0)
    {
        ReleaseDAC();
        return CHIP_NO_ERROR;
    }

    VerifyOrReturnError(dac.size() <= Credentials::kMaxDERCertLength, CHIP_ERROR_INVALID_ARGUMENT);
    if (mDACLen != 0)
    {
        ReleaseDAC();
    }

    VerifyOrReturnError(CanCastTo<uint16_t>(dac.size()), CHIP_ERROR_INVALID_ARGUMENT);
    if (mDAC == nullptr)
    {
        mDAC = static_cast<uint8_t *>(chip::Platform::MemoryAlloc(dac.size()));
    }
    VerifyOrReturnError(mDAC != nullptr, CHIP_ERROR_NO_MEMORY);
    mDACLen = static_cast<uint16_t>(dac.size());
    memcpy(mDAC, dac.data(), mDACLen);

    return CHIP_NO_ERROR;
}

void CommissioneeDeviceProxy::ReleasePAI()
{
    if (mPAI != nullptr)
    {
        chip::Platform::MemoryFree(mPAI);
    }
    mPAILen = 0;
    mPAI    = nullptr;
}

CHIP_ERROR CommissioneeDeviceProxy::SetPAI(const chip::ByteSpan & pai)
{
    if (pai.size() == 0)
    {
        ReleasePAI();
        return CHIP_NO_ERROR;
    }

    VerifyOrReturnError(pai.size() <= Credentials::kMaxDERCertLength, CHIP_ERROR_INVALID_ARGUMENT);
    if (mPAILen != 0)
    {
        ReleasePAI();
    }

    VerifyOrReturnError(CanCastTo<uint16_t>(pai.size()), CHIP_ERROR_INVALID_ARGUMENT);
    if (mPAI == nullptr)
    {
        mPAI = static_cast<uint8_t *>(chip::Platform::MemoryAlloc(pai.size()));
    }
    VerifyOrReturnError(mPAI != nullptr, CHIP_ERROR_NO_MEMORY);
    mPAILen = static_cast<uint16_t>(pai.size());
    memcpy(mPAI, pai.data(), mPAILen);

    return CHIP_NO_ERROR;
}

CommissioneeDeviceProxy::~CommissioneeDeviceProxy()
{
    ReleaseDAC();
    ReleasePAI();
}

CHIP_ERROR CommissioneeDeviceProxy::SetNOCCertBufferSize(size_t new_size)
{
    ReturnErrorCodeIf(new_size > sizeof(mNOCCertBuffer), CHIP_ERROR_INVALID_ARGUMENT);
    mNOCCertBufferSize = new_size;
    return CHIP_NO_ERROR;
}

CHIP_ERROR CommissioneeDeviceProxy::SetICACertBufferSize(size_t new_size)
{
    ReturnErrorCodeIf(new_size > sizeof(mICACertBuffer), CHIP_ERROR_INVALID_ARGUMENT);
    mICACertBufferSize = new_size;
    return CHIP_NO_ERROR;
}

} // namespace chip
<|endoftext|>


<commit_before>#include <stdio.h>
#include <sys/types.h>          // types needed by socket include files
#include <stdlib.h>             // free
#include <sys/socket.h>         // socket, bind, listen
#include <sys/un.h>             // sockaddr_un
#include <netinet/in.h>         // struct sockaddr_in
#include <netdb.h>              // gethostbyname
#include <arpa/inet.h>          // inet_ntoa
#include <netinet/tcp.h>        // TCP_NODELAY
#include <unistd.h>             // close
#include <fcntl.h>              // fcntl, F_SETFD
#include <errno.h>

#include "logMsg/logMsg.h"          // LM_X
#include "logMsg/traceLevels.h"

#include "au/TokenTaker.h"

#include "samson/common/MemoryTags.h"

#include "samson/common/status.h"

#include "samson/network/Packet.h"

#include "FileDescriptor.h" // Own interface

namespace samson {

    // Disconnect
    void FileDescriptor::close()
    {
        au::TokenTaker tt(&token);
        if( fd != -1 )
        {
            ::close(fd);
            fd = -1;
        }
    }

    bool FileDescriptor::isDisconnected()
    {
        return ( fd == -1 );
    }

    std::string FileDescriptor::str()
    {
        return au::str("FileDescriptor (fd=%d)" , fd );
    }

    Status FileDescriptor::okToSend(int tries, int tv_sec, int tv_usec)
    {
        int             fds;
        fd_set          wFds;
        struct timeval  timeVal;
        int             tryh;

        for (tryh = 0; tryh < tries; tryh++)
        {
            timeVal.tv_sec  = tv_sec;
            timeVal.tv_usec = tv_usec;

            // Connection previously closed by someone
            if( fd == -1 )
                return ConnectionClosed;

            FD_ZERO(&wFds);
            FD_SET(fd, &wFds);

            do
            {
                fds = select(fd + 1, NULL, &wFds, NULL, &timeVal);
            } while ((fds == -1) && (errno == EINTR));

            if ((fds == 1) && (FD_ISSET(fd, &wFds)))
                return OK;

            if( fds == -1 )
                LM_RE(Error, ("Select over fd %d: %s", fd , strerror(errno)));

            if( tryh > 3 )
                if( tryh%10 == 0 )
                    LM_W(("Problems to send to %s (%d/%d secs)", name.c_str() , tryh, tries ));
        }

        return Timeout;
    }

    Status FileDescriptor::partWrite( const void* dataP, int dataLen, const char* what, int retries, int tv_sec, int tv_usec)
    {
        int    nb;
        int    tot  = 0;
        char*  data = (char*) dataP;
        Status s;

        while (tot < dataLen)
        {
            if ((s = okToSend(retries, tv_sec, tv_usec)) != OK )
                LM_RE(s, ("Cannot write to '%s' after %d tries ( waiting %f seconds ) (fd %d)", name.c_str(), retries , (double) tv_sec + ((double)tv_usec/1000000.0)  , fd ));

            nb = ::write(fd, &data[tot], dataLen - tot);
            if (nb == -1)
                LM_RE(WriteError, ("error writing to '%s' (fd: %d): %s", name.c_str(), fd, strerror(errno)));
            else if (nb == 0)
                LM_RE(WriteError, ("part-write written ZERO bytes to '%s' (total: %d)", name.c_str(), tot));

            tot += nb;
        }

        LM_WRITES("Someone", what, data, tot, LmfByte);
        return OK;
    }

    Status FileDescriptor::writePacket( Packet * packetP , size_t *size )
    {
        Status           s;
        Message::Header  header;

        *size =0;

        LM_T(LmtSocketConnection, ("Sending Packet '%s' to %s " , packetP->str().c_str() , name.c_str() ));

        //
        // Preparing header
        //

        memset(&header, 0, sizeof(header));

        header.code        = packetP->msgCode;
        header.gbufLen     = packetP->message->ByteSize();
        header.kvDataLen   = packetP->buffer?packetP->buffer->getSize():0;

        // Set magic number
        header.setMagicNumber();

        //
        // Sending header
        //
        s = partWrite( &header, sizeof(header), "header");
        *size += sizeof(header);

        if (s != OK)
        {
            if (packetP->buffer != NULL)
                engine::MemoryManager::shared()->destroyBuffer(packetP->buffer);
            if (packetP != NULL)
                delete packetP;
            LM_RE(s, ("partWrite:header(%s): %s", name.c_str(), status(s)));
        }

        //
        // Sending Google Protocol Buffer
        //
        if ((packetP != NULL) && (packetP->message != NULL) && (packetP->message->ByteSize() != 0))
        {
            char* outputVec;

            outputVec = (char*) malloc(header.gbufLen + 2);
            if (outputVec == NULL)
                LM_XP(1, ("malloc(%d)", header.gbufLen));

            if (packetP->message->SerializeToArray(outputVec, header.gbufLen) == false)
                LM_X(1, ("SerializeToArray failed"));

            size_t size_message = packetP->message->ByteSize();
            s = partWrite( outputVec, size_message, "Google Protocol Buffer");
            free(outputVec);
            if (s != OK)
                LM_RE(s, ("partWrite:GoogleProtocolBuffer(): %s", status(s)));
            *size += size_message;

        }

        if (packetP && (packetP->buffer != 0))
        {
            s = partWrite( packetP->buffer->getData(), packetP->buffer->getSize(), "KV data");
            if (s != OK)
                LM_RE(s, ("partWrite returned %d and not the expected %d", s, packetP->buffer->getSize()));
            *size += packetP->buffer->getSize();
        }

        if (packetP != NULL)
        {
            if (packetP->buffer != NULL)
                engine::MemoryManager::shared()->destroyBuffer(packetP->buffer);
            delete packetP;
        }

        return OK;
    }

    Status FileDescriptor::msgAwait( int secs, int usecs, const char* what )
    {
        struct timeval  tv;
        struct timeval* tvP;
        int             fds;
        fd_set          rFds;

        do
        {
            if (secs == -1)
                tvP = NULL;
            else
            {
                tv.tv_sec  = secs;
                tv.tv_usec = usecs;

                tvP        = &tv;
            }

            FD_ZERO(&rFds);
            FD_SET(fd, &rFds);
            fds = select(fd + 1, &rFds, NULL, NULL, tvP);
        } while ((fds == -1) && (errno == EINTR));

        if (fds == -1)
        {
            //LM_RP(SelectError, ("select error awaiting '%s' from '%s", what, name.c_str()));
            return SelectError;
        }
        else if (fds == 0)
        {
            //LM_RE(Timeout, ("timeout awaiting '%s' from '%s' (%d.%06d seconds)", what, host.c_str(), secs, usecs));
            return Timeout;
        }
        else if ((fds > 0) && (!FD_ISSET(fd, &rFds)))
            LM_RE(Error, ("some other fd has a read pending - this is impossible ! (awaiting '%s' from '%s')", what, name.c_str()));
        else if ((fds > 0) && (FD_ISSET(fd, &rFds)))
            return OK;

        LM_X(1, ("Very strange error awaiting '%s' from '%s'", what, name.c_str()));

        return Error;
    }

    Status FileDescriptor::writeLine( const char* line, int retries, int tv_sec, int tv_usec)
    {
        Status s = partWrite(line, strlen(line), "write line", retries, tv_sec, tv_usec);

        if (s!= OK)
            close();

        return s;
    }

    Status FileDescriptor::readBuffer(char* line, size_t max_size, int tmoSecs)
    {
        int nb;
        nb = read(fd, line, max_size);

        if (nb > 0)
            return OK;
        return ReadError;
    }

    Status FileDescriptor::readLine( char* line, size_t max_size , int max_seconds )
    {
        au::Cronometer c;
        size_t tot = 0;

        while (true)
        {
            // Read a byte
            Status s = partRead(line + tot , 1 , "web line character", max_seconds - c.diffTimeInSeconds() );

            if( s!= OK )
            {
                close();
                return s;
            }

            if ( ( line[tot] == '\n' ) || ( line[tot] == '\r' ) )
            {
                line[tot] = '\0';
                LM_READS("REST Interface", "Web Line", line, tot, LmfByte);
                return OK;
            }

            // A new letter has been read
            tot++;

            // Check excesive line
            if( tot >= ( max_size - 2 ) )
                return Error;
        }

    }

    Status FileDescriptor::partRead( void* vbuf, size_t bufLen , const char* what , int max_seconds , size_t * read_size )
    {
        size_t  tot = 0;
        Status   s;
        char*    buf = (char*) vbuf;

        // Cronometer to measure absolute max time
        au::Cronometer cronometer;

        while (tot < bufLen)
        {
            ssize_t nb;

            // Wait until OK to read
            do
            {
                s = msgAwait(1, 0, what); // Continous try with 1 second timeout to check max_seconds

                if( ( s!= OK ) && ( s!= Timeout ) )
                {
                    if ( read_size )
                        *read_size = tot;
                    return s; // Different error, just report
                }

                // Report timeout if max seconds is excedded
                if( cronometer.diffTime() > max_seconds )
                {
                    if ( read_size )
                        *read_size = tot;
                    return Timeout;
                }

            } while ( s!= OK );

            // Read call
            nb = read(fd, (void*) &buf[tot] , bufLen - tot);

            // Report connection closed if so
            if (nb == -1)
            {
                if (errno == EBADF)
                    LM_RE(ConnectionClosed, ("read: %s (treating as Connection Closed), expecting '%s' from %s"
                                             , strerror(errno), what, name.c_str()));

                LM_RE(ConnectionClosed, ("read: %s, expecting '%s' from %s", strerror(errno), what, name.c_str()));
            }
            else if (nb == 0)
            {
                if ( read_size )
                    *read_size = tot;
                return ConnectionClosed;
            }

            tot += nb;
        }

        if ( read_size )
            *read_size = tot;

        return OK;
    }

    Status FileDescriptor::readPacket( Packet * packetP , size_t *size )
    {
        Status s;
        Message::Header  header;

        LM_T(LmtSocketConnection, ("SocketConnection %s: Reading packet" , str().c_str() ));
        *size = 0;

        s = partRead(&header, sizeof(Message::Header), "Header" , 300 );  // Timeout 300 secs for next packet
        if (s != OK)
        {
            close();
            return s;
        }
        *size += sizeof(Message::Header);

        // Check header
        if ( !header.check() )
        {
            close();
            return WrongPacketHeader;
        }

        packetP->msgCode = header.code;

        if (header.gbufLen != 0)
        {
            char* dataP = (char*) calloc(1, header.gbufLen + 1);

            s = partRead(dataP, header.gbufLen, "Google Protocol Buffer" , 300);
            if (s != OK)
            {
                close();
                free(dataP);
                return s;
            }
            *size += header.gbufLen;

            LM_T(LmtSocketConnection, ("Read %d bytes of GOOGLE DATA from '%s'", header.gbufLen, name.c_str() ));

            // Decode the google protocol buffer message
            packetP->message->ParseFromArray(dataP, header.gbufLen);

            if ( packetP->message->IsInitialized() == false )
            {
                LM_E(("Error parsing Google Protocol Buffer of %d bytes because a message %s is not initialized!",
                      header.gbufLen, samson::Message::messageCode(header.code)));

                close();
                return ErrorParsingGoogleProtocolBuffers;
            }

            free(dataP);
        }

        if (header.kvDataLen != 0)
        {
            // Alloc a buffer to read buffer of data
            std::string buffer_name = au::str("Network Buffer from %s" , name.c_str() );
            packetP->buffer = engine::MemoryManager::shared()->newBuffer( buffer_name , "network" , header.kvDataLen , 0.9 );

            char*  kvBuf  = packetP->buffer->getData();
            s = partRead(kvBuf, header.kvDataLen , "Key-Value Data" , 300);

            if (s != OK)
            {
                // Release buffer
                engine::MemoryManager::shared()->destroyBuffer( packetP->buffer );
                packetP->buffer = NULL;

                close();
                return s;
            }
            LM_T(LmtSocketConnection, ("Read %d bytes of KV DATA from '%s'", header.kvDataLen, name.c_str()));
            *size += header.kvDataLen;

            packetP->buffer->setSize( header.kvDataLen );
        }

        return OK;
    }

}<commit_msg>Minor bug fixed. Detect closed connections in read thread <commit_after>#include <stdio.h>
#include <sys/types.h>          // types needed by socket include files
#include <stdlib.h>             // free
#include <sys/socket.h>         // socket, bind, listen
#include <sys/un.h>             // sockaddr_un
#include <netinet/in.h>         // struct sockaddr_in
#include <netdb.h>              // gethostbyname
#include <arpa/inet.h>          // inet_ntoa
#include <netinet/tcp.h>        // TCP_NODELAY
#include <unistd.h>             // close
#include <fcntl.h>              // fcntl, F_SETFD
#include <errno.h>

#include "logMsg/logMsg.h"          // LM_X
#include "logMsg/traceLevels.h"

#include "au/TokenTaker.h"

#include "samson/common/MemoryTags.h"

#include "samson/common/status.h"

#include "samson/network/Packet.h"

#include "FileDescriptor.h" // Own interface

namespace samson {

    // Disconnect
    void FileDescriptor::close()
    {
        au::TokenTaker tt(&token);
        if( fd != -1 )
        {
            ::close(fd);
            fd = -1;
        }
    }

    bool FileDescriptor::isDisconnected()
    {
        return ( fd == -1 );
    }

    std::string FileDescriptor::str()
    {
        return au::str("FileDescriptor (fd=%d)" , fd );
    }

    Status FileDescriptor::okToSend(int tries, int tv_sec, int tv_usec)
    {
        int             fds;
        fd_set          wFds;
        struct timeval  timeVal;
        int             tryh;

        for (tryh = 0; tryh < tries; tryh++)
        {
            timeVal.tv_sec  = tv_sec;
            timeVal.tv_usec = tv_usec;

            // Connection previously closed by someone
            if( fd == -1 )
                return ConnectionClosed;

            FD_ZERO(&wFds);
            FD_SET(fd, &wFds);

            do
            {
                fds = select(fd + 1, NULL, &wFds, NULL, &timeVal);
            } while ((fds == -1) && (errno == EINTR));

            if ((fds == 1) && (FD_ISSET(fd, &wFds)))
                return OK;

            if( fds == -1 )
                LM_RE(Error, ("Select over fd %d: %s", fd , strerror(errno)));

            if( tryh > 3 )
                if( tryh%10 == 0 )
                    LM_W(("Problems to send to %s (%d/%d secs)", name.c_str() , tryh, tries ));
        }

        return Timeout;
    }

    Status FileDescriptor::partWrite( const void* dataP, int dataLen, const char* what, int retries, int tv_sec, int tv_usec)
    {
        int    nb;
        int    tot  = 0;
        char*  data = (char*) dataP;
        Status s;

        while (tot < dataLen)
        {
            if ((s = okToSend(retries, tv_sec, tv_usec)) != OK )
                LM_RE(s, ("Cannot write to '%s' after %d tries ( waiting %f seconds ) (fd %d)", name.c_str(), retries , (double) tv_sec + ((double)tv_usec/1000000.0)  , fd ));

            nb = ::write(fd, &data[tot], dataLen - tot);
            if (nb == -1)
                LM_RE(WriteError, ("error writing to '%s' (fd: %d): %s", name.c_str(), fd, strerror(errno)));
            else if (nb == 0)
                LM_RE(WriteError, ("part-write written ZERO bytes to '%s' (total: %d)", name.c_str(), tot));

            tot += nb;
        }

        LM_WRITES("Someone", what, data, tot, LmfByte);
        return OK;
    }

    Status FileDescriptor::writePacket( Packet * packetP , size_t *size )
    {
        Status           s;
        Message::Header  header;

        *size =0;

        LM_T(LmtSocketConnection, ("Sending Packet '%s' to %s " , packetP->str().c_str() , name.c_str() ));

        //
        // Preparing header
        //

        memset(&header, 0, sizeof(header));

        header.code        = packetP->msgCode;
        header.gbufLen     = packetP->message->ByteSize();
        header.kvDataLen   = packetP->buffer?packetP->buffer->getSize():0;

        // Set magic number
        header.setMagicNumber();

        //
        // Sending header
        //
        s = partWrite( &header, sizeof(header), "header");
        *size += sizeof(header);

        if (s != OK)
        {
            if (packetP->buffer != NULL)
                engine::MemoryManager::shared()->destroyBuffer(packetP->buffer);
            if (packetP != NULL)
                delete packetP;
            LM_RE(s, ("partWrite:header(%s): %s", name.c_str(), status(s)));
        }

        //
        // Sending Google Protocol Buffer
        //
        if ((packetP != NULL) && (packetP->message != NULL) && (packetP->message->ByteSize() != 0))
        {
            char* outputVec;

            outputVec = (char*) malloc(header.gbufLen + 2);
            if (outputVec == NULL)
                LM_XP(1, ("malloc(%d)", header.gbufLen));

            if (packetP->message->SerializeToArray(outputVec, header.gbufLen) == false)
                LM_X(1, ("SerializeToArray failed"));

            size_t size_message = packetP->message->ByteSize();
            s = partWrite( outputVec, size_message, "Google Protocol Buffer");
            free(outputVec);
            if (s != OK)
                LM_RE(s, ("partWrite:GoogleProtocolBuffer(): %s", status(s)));
            *size += size_message;

        }

        if (packetP && (packetP->buffer != 0))
        {
            s = partWrite( packetP->buffer->getData(), packetP->buffer->getSize(), "KV data");
            if (s != OK)
                LM_RE(s, ("partWrite returned %d and not the expected %d", s, packetP->buffer->getSize()));
            *size += packetP->buffer->getSize();
        }

        if (packetP != NULL)
        {
            if (packetP->buffer != NULL)
                engine::MemoryManager::shared()->destroyBuffer(packetP->buffer);
            delete packetP;
        }

        return OK;
    }

    Status FileDescriptor::msgAwait( int secs, int usecs, const char* what )
    {
        struct timeval  tv;
        struct timeval* tvP;
        int             fds;
        fd_set          rFds;

        do
        {
            if (secs == -1)
                tvP = NULL;
            else
            {
                tv.tv_sec  = secs;
                tv.tv_usec = usecs;

                tvP        = &tv;
            }

            FD_ZERO(&rFds);
            FD_SET(fd, &rFds);
            fds = select(fd + 1, &rFds, NULL, NULL, tvP);
        } while ((fds == -1) && (errno == EINTR));

        if (fds == -1)
        {
            //LM_RP(SelectError, ("select error awaiting '%s' from '%s", what, name.c_str()));
            return SelectError;
        }
        else if (fds == 0)
        {
            //LM_RE(Timeout, ("timeout awaiting '%s' from '%s' (%d.%06d seconds)", what, host.c_str(), secs, usecs));
            return Timeout;
        }
        else if ((fds > 0) && (!FD_ISSET(fd, &rFds)))
            LM_RE(Error, ("some other fd has a read pending - this is impossible ! (awaiting '%s' from '%s')", what, name.c_str()));
        else if ((fds > 0) && (FD_ISSET(fd, &rFds)))
            return OK;

        LM_X(1, ("Very strange error awaiting '%s' from '%s'", what, name.c_str()));

        return Error;
    }

    Status FileDescriptor::writeLine( const char* line, int retries, int tv_sec, int tv_usec)
    {
        Status s = partWrite(line, strlen(line), "write line", retries, tv_sec, tv_usec);

        if (s!= OK)
            close();

        return s;
    }

    Status FileDescriptor::readBuffer(char* line, size_t max_size, int tmoSecs)
    {
        int nb;
        nb = read(fd, line, max_size);

        if (nb > 0)
            return OK;
        return ReadError;
    }

    Status FileDescriptor::readLine( char* line, size_t max_size , int max_seconds )
    {
        au::Cronometer c;
        size_t tot = 0;

        while (true)
        {
            // Read a byte
            Status s = partRead(line + tot , 1 , "web line character", max_seconds - c.diffTimeInSeconds() );

            if( s!= OK )
            {
                close();
                return s;
            }

            if ( ( line[tot] == '\n' ) || ( line[tot] == '\r' ) )
            {
                line[tot] = '\0';
                LM_READS("REST Interface", "Web Line", line, tot, LmfByte);
                return OK;
            }

            // A new letter has been read
            tot++;

            // Check excesive line
            if( tot >= ( max_size - 2 ) )
                return Error;
        }

    }

    Status FileDescriptor::partRead( void* vbuf, size_t bufLen , const char* what , int max_seconds , size_t * read_size )
    {
        size_t  tot = 0;
        Status   s;
        char*    buf = (char*) vbuf;

        // Cronometer to measure absolute max time
        au::Cronometer cronometer;

        while (tot < bufLen)
        {
            ssize_t nb;

            // Wait until OK to read
            do
            {
                if( fd == -1 )
                    return ConnectionClosed;

                s = msgAwait(1, 0, what); // Continous try with 1 second timeout to check max_seconds

                if( ( s!= OK ) && ( s!= Timeout ) )
                {
                    if ( read_size )
                        *read_size = tot;
                    return s; // Different error, just report
                }

                // Report timeout if max seconds is excedded
                if( cronometer.diffTime() > max_seconds )
                {
                    if ( read_size )
                        *read_size = tot;
                    return Timeout;
                }

            } while ( s!= OK );

            // Read call
            nb = read(fd, (void*) &buf[tot] , bufLen - tot);

            // Report connection closed if so
            if (nb == -1)
            {
                if (errno == EBADF)
                    LM_RE(ConnectionClosed, ("read: %s (treating as Connection Closed), expecting '%s' from %s"
                                             , strerror(errno), what, name.c_str()));

                LM_RE(ConnectionClosed, ("read: %s, expecting '%s' from %s", strerror(errno), what, name.c_str()));
            }
            else if (nb == 0)
            {
                if ( read_size )
                    *read_size = tot;
                return ConnectionClosed;
            }

            tot += nb;
        }

        if ( read_size )
            *read_size = tot;

        return OK;
    }

    Status FileDescriptor::readPacket( Packet * packetP , size_t *size )
    {
        Status s;
        Message::Header  header;

        LM_T(LmtSocketConnection, ("SocketConnection %s: Reading packet" , str().c_str() ));
        *size = 0;

        s = partRead(&header, sizeof(Message::Header), "Header" , 300 );  // Timeout 300 secs for next packet
        if (s != OK)
        {
            close();
            return s;
        }
        *size += sizeof(Message::Header);

        // Check header
        if ( !header.check() )
        {
            close();
            return WrongPacketHeader;
        }

        packetP->msgCode = header.code;

        if (header.gbufLen != 0)
        {
            char* dataP = (char*) calloc(1, header.gbufLen + 1);

            s = partRead(dataP, header.gbufLen, "Google Protocol Buffer" , 300);
            if (s != OK)
            {
                close();
                free(dataP);
                return s;
            }
            *size += header.gbufLen;

            LM_T(LmtSocketConnection, ("Read %d bytes of GOOGLE DATA from '%s'", header.gbufLen, name.c_str() ));

            // Decode the google protocol buffer message
            packetP->message->ParseFromArray(dataP, header.gbufLen);

            if ( packetP->message->IsInitialized() == false )
            {
                LM_E(("Error parsing Google Protocol Buffer of %d bytes because a message %s is not initialized!",
                      header.gbufLen, samson::Message::messageCode(header.code)));

                close();
                return ErrorParsingGoogleProtocolBuffers;
            }

            free(dataP);
        }

        if (header.kvDataLen != 0)
        {
            // Alloc a buffer to read buffer of data
            std::string buffer_name = au::str("Network Buffer from %s" , name.c_str() );
            packetP->buffer = engine::MemoryManager::shared()->newBuffer( buffer_name , "network" , header.kvDataLen , 0.9 );

            char*  kvBuf  = packetP->buffer->getData();
            s = partRead(kvBuf, header.kvDataLen , "Key-Value Data" , 300);

            if (s != OK)
            {
                // Release buffer
                engine::MemoryManager::shared()->destroyBuffer( packetP->buffer );
                packetP->buffer = NULL;

                close();
                return s;
            }
            LM_T(LmtSocketConnection, ("Read %d bytes of KV DATA from '%s'", header.kvDataLen, name.c_str()));
            *size += header.kvDataLen;

            packetP->buffer->setSize( header.kvDataLen );
        }

        return OK;
    }

}<|endoftext|>


<commit_before>#include <algorithm>
#include <cfloat>
#include <vector>

#include "caffe/layer.hpp"
#include "caffe/layer_factory.hpp"
#include "caffe/util/math_functions.hpp"
#include "caffe/vision_layers.hpp"

namespace caffe {

template <typename Dtype>
void SoftmaxWithLossLayer<Dtype>::LayerSetUp(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
  LossLayer<Dtype>::LayerSetUp(bottom, top);
  LayerParameter softmax_param(this->layer_param_);
  softmax_param.set_type("Softmax");
  softmax_layer_ = LayerRegistry<Dtype>::CreateLayer(softmax_param);
  softmax_bottom_vec_.clear();
  softmax_bottom_vec_.push_back(bottom[0]);
  softmax_top_vec_.clear();
  softmax_top_vec_.push_back(&prob_);
  softmax_layer_->SetUp(softmax_bottom_vec_, softmax_top_vec_);

  has_ignore_label_ =
    this->layer_param_.loss_param().has_ignore_label();
  if (has_ignore_label_) {
    ignore_label_ = this->layer_param_.loss_param().ignore_label();
  }
  normalize_ = this->layer_param_.loss_param().normalize();
}

template <typename Dtype>
void SoftmaxWithLossLayer<Dtype>::Reshape(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
  LossLayer<Dtype>::Reshape(bottom, top);
  softmax_layer_->Reshape(softmax_bottom_vec_, softmax_top_vec_);
  softmax_axis_ = this->layer_param_.softmax_param().axis();
  outer_num_ = bottom[0]->count(0, softmax_axis_);
  inner_num_ = bottom[0]->count(softmax_axis_ + 1);
  CHECK_EQ(outer_num_ * inner_num_, bottom[1]->count())
      << "Number of labels must match number of predictions; "
      << "e.g., if softmax axis == 1 and prediction shape is (N, C, H, W), "
      << "label count (number of labels) must be N*H*W, "
      << "with integer values in {0, 1, ..., C-1}.";
  if (top.size() >= 2) {
    // softmax output
    top[1]->ReshapeLike(*bottom[0]);
  }
}

template <typename Dtype>
void SoftmaxWithLossLayer<Dtype>::Forward_cpu(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
  // The forward pass computes the softmax prob values.
  softmax_layer_->Forward(softmax_bottom_vec_, softmax_top_vec_);
  const Dtype* prob_data = prob_.cpu_data();
  const Dtype* label = bottom[1]->cpu_data();
  int dim = prob_.count() / outer_num_;
  int count = 0;
  Dtype loss = 0;
  for (int i = 0; i < outer_num_; ++i) {
    for (int j = 0; j < inner_num_; j++) {
      const int label_value = static_cast<int>(label[i * inner_num_ + j]);
      if (has_ignore_label_ && label_value == ignore_label_) {
        continue;
      }
      DCHECK_GE(label_value, 0);
      DCHECK_LT(label_value, prob_.shape(softmax_axis_));
      loss -= log(std::max(prob_data[i * dim + label_value * inner_num_ + j],
                           Dtype(FLT_MIN)));
      ++count;
    }
  }
  if (normalize_) {
    top[0]->mutable_cpu_data()[0] = loss / count;
  } else {
    top[0]->mutable_cpu_data()[0] = loss / outer_num_;
  }
  if (top.size() == 2) {
    top[1]->ShareData(prob_);
  }
}

template <typename Dtype>
void SoftmaxWithLossLayer<Dtype>::Backward_cpu(const vector<Blob<Dtype>*>& top,
    const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {
  if (propagate_down[1]) {
    LOG(FATAL) << this->type()
               << " Layer cannot backpropagate to label inputs.";
  }
  if (propagate_down[0]) {
    Dtype* bottom_diff = bottom[0]->mutable_cpu_diff();
    const Dtype* prob_data = prob_.cpu_data();
    caffe_copy(prob_.count(), prob_data, bottom_diff);
    const Dtype* label = bottom[1]->cpu_data();
    int dim = prob_.count() / outer_num_;
    int count = 0;
    for (int i = 0; i < outer_num_; ++i) {
      for (int j = 0; j < inner_num_; ++j) {
        const int label_value = static_cast<int>(label[i * inner_num_ + j]);
        if (has_ignore_label_ && label_value == ignore_label_) {
          for (int c = 0; c < bottom[0]->shape(softmax_axis_); ++c) {
            bottom_diff[i * dim + c * inner_num_ + j] = 0;
          }
        } else {
          bottom_diff[i * dim + label_value * inner_num_ + j] -= 1;
          ++count;
        }
      }
    }
    // Scale gradient
    const Dtype loss_weight = top[0]->cpu_diff()[0];
    if (normalize_) {
      caffe_scal(prob_.count(), loss_weight / count, bottom_diff);
    } else {
      caffe_scal(prob_.count(), loss_weight / outer_num_, bottom_diff);
    }
  }
}

#ifdef CPU_ONLY
STUB_GPU(SoftmaxWithLossLayer);
#endif

INSTANTIATE_CLASS(SoftmaxWithLossLayer);
REGISTER_LAYER_CLASS(SoftmaxWithLoss);

}  // namespace caffe
<commit_msg>SoftmaxLossLayer fix: canonicalize input axis<commit_after>#include <algorithm>
#include <cfloat>
#include <vector>

#include "caffe/layer.hpp"
#include "caffe/layer_factory.hpp"
#include "caffe/util/math_functions.hpp"
#include "caffe/vision_layers.hpp"

namespace caffe {

template <typename Dtype>
void SoftmaxWithLossLayer<Dtype>::LayerSetUp(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
  LossLayer<Dtype>::LayerSetUp(bottom, top);
  LayerParameter softmax_param(this->layer_param_);
  softmax_param.set_type("Softmax");
  softmax_layer_ = LayerRegistry<Dtype>::CreateLayer(softmax_param);
  softmax_bottom_vec_.clear();
  softmax_bottom_vec_.push_back(bottom[0]);
  softmax_top_vec_.clear();
  softmax_top_vec_.push_back(&prob_);
  softmax_layer_->SetUp(softmax_bottom_vec_, softmax_top_vec_);

  has_ignore_label_ =
    this->layer_param_.loss_param().has_ignore_label();
  if (has_ignore_label_) {
    ignore_label_ = this->layer_param_.loss_param().ignore_label();
  }
  normalize_ = this->layer_param_.loss_param().normalize();
}

template <typename Dtype>
void SoftmaxWithLossLayer<Dtype>::Reshape(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
  LossLayer<Dtype>::Reshape(bottom, top);
  softmax_layer_->Reshape(softmax_bottom_vec_, softmax_top_vec_);
  softmax_axis_ =
      bottom[0]->CanonicalAxisIndex(this->layer_param_.softmax_param().axis());
  outer_num_ = bottom[0]->count(0, softmax_axis_);
  inner_num_ = bottom[0]->count(softmax_axis_ + 1);
  CHECK_EQ(outer_num_ * inner_num_, bottom[1]->count())
      << "Number of labels must match number of predictions; "
      << "e.g., if softmax axis == 1 and prediction shape is (N, C, H, W), "
      << "label count (number of labels) must be N*H*W, "
      << "with integer values in {0, 1, ..., C-1}.";
  if (top.size() >= 2) {
    // softmax output
    top[1]->ReshapeLike(*bottom[0]);
  }
}

template <typename Dtype>
void SoftmaxWithLossLayer<Dtype>::Forward_cpu(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
  // The forward pass computes the softmax prob values.
  softmax_layer_->Forward(softmax_bottom_vec_, softmax_top_vec_);
  const Dtype* prob_data = prob_.cpu_data();
  const Dtype* label = bottom[1]->cpu_data();
  int dim = prob_.count() / outer_num_;
  int count = 0;
  Dtype loss = 0;
  for (int i = 0; i < outer_num_; ++i) {
    for (int j = 0; j < inner_num_; j++) {
      const int label_value = static_cast<int>(label[i * inner_num_ + j]);
      if (has_ignore_label_ && label_value == ignore_label_) {
        continue;
      }
      DCHECK_GE(label_value, 0);
      DCHECK_LT(label_value, prob_.shape(softmax_axis_));
      loss -= log(std::max(prob_data[i * dim + label_value * inner_num_ + j],
                           Dtype(FLT_MIN)));
      ++count;
    }
  }
  if (normalize_) {
    top[0]->mutable_cpu_data()[0] = loss / count;
  } else {
    top[0]->mutable_cpu_data()[0] = loss / outer_num_;
  }
  if (top.size() == 2) {
    top[1]->ShareData(prob_);
  }
}

template <typename Dtype>
void SoftmaxWithLossLayer<Dtype>::Backward_cpu(const vector<Blob<Dtype>*>& top,
    const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {
  if (propagate_down[1]) {
    LOG(FATAL) << this->type()
               << " Layer cannot backpropagate to label inputs.";
  }
  if (propagate_down[0]) {
    Dtype* bottom_diff = bottom[0]->mutable_cpu_diff();
    const Dtype* prob_data = prob_.cpu_data();
    caffe_copy(prob_.count(), prob_data, bottom_diff);
    const Dtype* label = bottom[1]->cpu_data();
    int dim = prob_.count() / outer_num_;
    int count = 0;
    for (int i = 0; i < outer_num_; ++i) {
      for (int j = 0; j < inner_num_; ++j) {
        const int label_value = static_cast<int>(label[i * inner_num_ + j]);
        if (has_ignore_label_ && label_value == ignore_label_) {
          for (int c = 0; c < bottom[0]->shape(softmax_axis_); ++c) {
            bottom_diff[i * dim + c * inner_num_ + j] = 0;
          }
        } else {
          bottom_diff[i * dim + label_value * inner_num_ + j] -= 1;
          ++count;
        }
      }
    }
    // Scale gradient
    const Dtype loss_weight = top[0]->cpu_diff()[0];
    if (normalize_) {
      caffe_scal(prob_.count(), loss_weight / count, bottom_diff);
    } else {
      caffe_scal(prob_.count(), loss_weight / outer_num_, bottom_diff);
    }
  }
}

#ifdef CPU_ONLY
STUB_GPU(SoftmaxWithLossLayer);
#endif

INSTANTIATE_CLASS(SoftmaxWithLossLayer);
REGISTER_LAYER_CLASS(SoftmaxWithLoss);

}  // namespace caffe
<|endoftext|>