<commit_before>/*
 *  caosVM_agent.cpp
 *  openc2e
 *
 *  Created by Alyssa Milburn on Tue Jun 01 2004.
 *  Copyright (c) 2004 Alyssa Milburn. All rights reserved.
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License as published by the Free Software Foundation; either
 *  version 2 of the License, or (at your option) any later version.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 */

#include "caosVM.h"
#include "openc2e.h"
#include "Agent.h"
#include "AgentRef.h"
#include "World.h"
#include <iostream>
using std::cerr;

/**
 ELAS (command) elas (integer)
*/
void caosVM::c_ELAS() {
	VM_VERIFY_SIZE(1)
	VM_PARAM_INTEGER(elas)
	// TODO
}

/**
 MVTO (command) x (float) y (float)
*/
void caosVM::c_MVTO() {
	VM_VERIFY_SIZE(2)
	VM_PARAM_FLOAT(y)
	VM_PARAM_FLOAT(x)
	assert(targ);
	targ->moveTo(x, y);
}

/**
 VELX (variable)
*/
void caosVM::v_VELX() {
	VM_VERIFY_SIZE(0)
	assert(targ);
	result = targ->velx;
	result.setVariable(&targ->velx);
}

/**
 VELY (variable)
*/
void caosVM::v_VELY() {
	VM_VERIFY_SIZE(0)

	assert(targ);
	result = targ->vely;
	result.setVariable(&targ->vely);
}

/**
 OBST (float) direction (integer)

 returns distance from target agent to nearest wall that it might collide with in given direction
 (except right now it just gives the direction to the nearest wall at world edge - fuzzie)
*/
void caosVM::v_OBST() {
	VM_VERIFY_SIZE(1)
	VM_PARAM_INTEGER(direction) assert(direction >= 0); assert(direction <= 3);

	// TODO: fix 'might collide with' issue
	// note: this code is mostly copied from GRID - fuzzie

	// TODO: this code should calculate distance *from agent*, not *from centre point*, i expect
	// someone check with DS? - fuzzie

	float agentx = targ->x + (targ->getWidth() / 2);
	float agenty = targ->y + (targ->getHeight() / 2);
	Room *sourceroom = world.map.roomAt(agentx, agenty);
	if (!sourceroom) {
		// (should we REALLY check for it being in the room system, here?)
		cerr << targ->identify() << " tried using OBST but isn't in the room system!\n";
		result.setInt(-1);
		return;
	}

	int distance = 0;

	if ((direction == 0) || (direction == 1)) {
		int movement = (direction == 0 ? -1 : 1);

		int x = agentx;
		while (true) {
			x += movement;
			Room *r = world.map.roomAt(x, agenty);
			if (!r)
				break;
			distance++;
		}
	} else if ((direction == 2) || (direction == 3)) {
		int movement = (direction == 2 ? -1 : 1);

		int y = agenty;
		while (true) {
			y += movement;
			Room *r = world.map.roomAt(agentx, y);
			if (!r)
				break;
			distance++;
		}
	} else cerr << "OBST got an unknown direction!\n";

	result.setInt(distance);
}

/**
 TMVT (integer) x (float) y (float)

 returns 1 if TARG could move to (x, y) and still be in room system, otherwise returns 0
*/
void caosVM::v_TMVT() {
	VM_VERIFY_SIZE(2)
	VM_PARAM_FLOAT(y)
	VM_PARAM_FLOAT(x)

	assert(targ);
	// TODO: do this properly
	Room *r1 = world.map.roomAt(x, y);
	Room *r2 = world.map.roomAt(x + (targ->getWidth() / 2), y + (targ->getHeight() / 2));
	result.setInt((r1 && r2) ? 1 : 0);
}

/**
 TMVF (integer) x (float) y (float)

 returns 1 if target create could move foot to (x, y) and still be in room system, otherwise returns 0
*/
void caosVM::v_TMVF() {
	VM_VERIFY_SIZE(2)
	VM_PARAM_FLOAT(y)
	VM_PARAM_FLOAT(x)

	assert(targ);
	result.setInt(1); // TODO: don't hardcode
}

/**
 ACCG (command) accel (float)
*/
void caosVM::c_ACCG() {
	VM_VERIFY_SIZE(1)
	VM_PARAM_FLOAT(accel)

	assert(targ);
	targ->accg = accel;
}

/**
 ACCG (float)
*/
void caosVM::v_ACCG() {
	VM_VERIFY_SIZE(0)

	assert(targ);
	result.setFloat(targ->accg);
}

/**
 AERO (command) aero (float)
*/
void caosVM::c_AERO() {
	VM_VERIFY_SIZE(1)
	VM_PARAM_FLOAT(aero)

	assert(targ);
	targ->aero = aero;
}

/**
 AERO (float)
*/
void caosVM::v_AERO() {
	VM_VERIFY_SIZE(0)

	assert(targ);
	result.setFloat(targ->aero);
}

/**
 RELX (float) first (agent) second (agent)
*/
void caosVM::v_RELX() {
	VM_VERIFY_SIZE(2)
	VM_PARAM_AGENT(second)
	VM_PARAM_AGENT(first)

	// TODO: should we divide by 2.0 to get float answer?
	float one = first->x + (first->getWidth() / 2);
	float two = second->x + (second->getWidth() / 2);

	result.setFloat(two - one);
}

/**
 RELY (float) first (agent) second (agent)
*/
void caosVM::v_RELY() {
	VM_VERIFY_SIZE(2)
	VM_PARAM_AGENT(second)
	VM_PARAM_AGENT(first)

	// TODO: should we divide by 2.0 to get float answer? check it with the second lift in the norn terrarium..
	float one = first->y + (first->getHeight() / 2);
	float two = second->y + (second->getHeight() / 2);

	result.setFloat(two - one);
}

/**
 VELO (command) xvel (float) yvel (float)
*/
void caosVM::c_VELO() {
	VM_VERIFY_SIZE(2)
	VM_PARAM_FLOAT(vely)
	VM_PARAM_FLOAT(velx)

	assert(targ);
	targ->velx.reset();
	targ->velx.setFloat(velx);
	targ->vely.reset();
	targ->vely.setFloat(vely);
}

/**
 MVSF (command) x (float) y (float)

 move the target agent to an area inside the room system at about (x, y)
*/
void caosVM::c_MVSF() {
	// TODO
	c_MVTO();
}

/**
 FRIC (float)
*/
void caosVM::v_FRIC() {
	VM_VERIFY_SIZE(0)

	assert(targ);
	result.setFloat(targ->friction);
}

/**
 FRIC (command) friction (integer)
*/
void caosVM::c_FRIC() {
	VM_VERIFY_SIZE(1)
	VM_PARAM_INTEGER(friction) caos_assert(friction >= 0); caos_assert(friction <= 100);

	assert(targ);
	targ->friction = friction;
}

/**
 FALL (integer)

 Return 1 if TARG is moving due to gravity, 0 otherwise
*/
void caosVM::v_FALL() {
	VM_VERIFY_SIZE(0)
	caos_assert(targ);

	float accg = targ->accg;
	float vely = targ->vely.getFloat();
	if (accg != 0 && vely != 0) {
		// XXX: this probably isn't quite correct, but it's close enough for
		// now. Hook it into the physics system properly later
		result.setInt(1);
	} else
		result.setInt(0);
}
<commit_msg>make OBST be a bit more intelligent for speed reasons<commit_after>/*
 *  caosVM_agent.cpp
 *  openc2e
 *
 *  Created by Alyssa Milburn on Tue Jun 01 2004.
 *  Copyright (c) 2004 Alyssa Milburn. All rights reserved.
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License as published by the Free Software Foundation; either
 *  version 2 of the License, or (at your option) any later version.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 */

#include "caosVM.h"
#include "openc2e.h"
#include "Agent.h"
#include "AgentRef.h"
#include "World.h"
#include <iostream>
using std::cerr;

/**
 ELAS (command) elas (integer)
*/
void caosVM::c_ELAS() {
	VM_VERIFY_SIZE(1)
	VM_PARAM_INTEGER(elas)
	// TODO
}

/**
 MVTO (command) x (float) y (float)
*/
void caosVM::c_MVTO() {
	VM_VERIFY_SIZE(2)
	VM_PARAM_FLOAT(y)
	VM_PARAM_FLOAT(x)
	assert(targ);
	targ->moveTo(x, y);
}

/**
 VELX (variable)
*/
void caosVM::v_VELX() {
	VM_VERIFY_SIZE(0)
	assert(targ);
	result = targ->velx;
	result.setVariable(&targ->velx);
}

/**
 VELY (variable)
*/
void caosVM::v_VELY() {
	VM_VERIFY_SIZE(0)

	assert(targ);
	result = targ->vely;
	result.setVariable(&targ->vely);
}

/**
 OBST (float) direction (integer)

 returns distance from target agent to nearest wall that it might collide with in given direction
 (except right now it just gives the direction to the nearest wall at world edge - fuzzie)
*/
void caosVM::v_OBST() {
	VM_VERIFY_SIZE(1)
	VM_PARAM_INTEGER(direction) assert(direction >= 0); assert(direction <= 3);

	// TODO: fix 'might collide with' issue
	// note: this code is mostly copied from GRID - fuzzie

	// TODO: this code should calculate distance *from agent*, not *from centre point*, i expect
	// someone check with DS? - fuzzie

	float agentx = targ->x + (targ->getWidth() / 2);
	float agenty = targ->y + (targ->getHeight() / 2);
	Room *sourceroom = world.map.roomAt(agentx, agenty);
	if (!sourceroom) {
		// (should we REALLY check for it being in the room system, here?)
		cerr << targ->identify() << " tried using OBST but isn't in the room system!\n";
		result.setInt(-1);
		return;
	}

	int distance = 0;

	if ((direction == 0) || (direction == 1)) {
		int movement = (direction == 0 ? -1 : 1);

		int x = agentx;
		Room *r = 0;
		while (true) {
			x += movement;
			if (r) {
				if (!r->containsPoint(x, agenty))
					r = world.map.roomAt(x, agenty);
			} else {
				r = world.map.roomAt(x, agenty);
			}
			if (!r)
				break;
			distance++;
		}
	} else if ((direction == 2) || (direction == 3)) {
		int movement = (direction == 2 ? -1 : 1);

		int y = agenty;
		Room *r = 0;
		while (true) {
			y += movement;
			if (r) {
				if (!r->containsPoint(agentx, y))
					r = world.map.roomAt(agentx, y);
			} else {
				r = world.map.roomAt(agentx, y);
			}
			if (!r)
				break;
			distance++;
		}
	} else cerr << "OBST got an unknown direction!\n";

	result.setInt(distance);
}

/**
 TMVT (integer) x (float) y (float)

 returns 1 if TARG could move to (x, y) and still be in room system, otherwise returns 0
*/
void caosVM::v_TMVT() {
	VM_VERIFY_SIZE(2)
	VM_PARAM_FLOAT(y)
	VM_PARAM_FLOAT(x)

	assert(targ);
	// TODO: do this properly
	Room *r1 = world.map.roomAt(x, y);
	Room *r2 = world.map.roomAt(x + (targ->getWidth() / 2), y + (targ->getHeight() / 2));
	result.setInt((r1 && r2) ? 1 : 0);
}

/**
 TMVF (integer) x (float) y (float)

 returns 1 if target create could move foot to (x, y) and still be in room system, otherwise returns 0
*/
void caosVM::v_TMVF() {
	VM_VERIFY_SIZE(2)
	VM_PARAM_FLOAT(y)
	VM_PARAM_FLOAT(x)

	assert(targ);
	result.setInt(1); // TODO: don't hardcode
}

/**
 ACCG (command) accel (float)
*/
void caosVM::c_ACCG() {
	VM_VERIFY_SIZE(1)
	VM_PARAM_FLOAT(accel)

	assert(targ);
	targ->accg = accel;
}

/**
 ACCG (float)
*/
void caosVM::v_ACCG() {
	VM_VERIFY_SIZE(0)

	assert(targ);
	result.setFloat(targ->accg);
}

/**
 AERO (command) aero (float)
*/
void caosVM::c_AERO() {
	VM_VERIFY_SIZE(1)
	VM_PARAM_FLOAT(aero)

	assert(targ);
	targ->aero = aero;
}

/**
 AERO (float)
*/
void caosVM::v_AERO() {
	VM_VERIFY_SIZE(0)

	assert(targ);
	result.setFloat(targ->aero);
}

/**
 RELX (float) first (agent) second (agent)
*/
void caosVM::v_RELX() {
	VM_VERIFY_SIZE(2)
	VM_PARAM_AGENT(second)
	VM_PARAM_AGENT(first)

	// TODO: should we divide by 2.0 to get float answer?
	float one = first->x + (first->getWidth() / 2);
	float two = second->x + (second->getWidth() / 2);

	result.setFloat(two - one);
}

/**
 RELY (float) first (agent) second (agent)
*/
void caosVM::v_RELY() {
	VM_VERIFY_SIZE(2)
	VM_PARAM_AGENT(second)
	VM_PARAM_AGENT(first)

	// TODO: should we divide by 2.0 to get float answer? check it with the second lift in the norn terrarium..
	float one = first->y + (first->getHeight() / 2);
	float two = second->y + (second->getHeight() / 2);

	result.setFloat(two - one);
}

/**
 VELO (command) xvel (float) yvel (float)
*/
void caosVM::c_VELO() {
	VM_VERIFY_SIZE(2)
	VM_PARAM_FLOAT(vely)
	VM_PARAM_FLOAT(velx)

	assert(targ);
	targ->velx.reset();
	targ->velx.setFloat(velx);
	targ->vely.reset();
	targ->vely.setFloat(vely);
}

/**
 MVSF (command) x (float) y (float)

 move the target agent to an area inside the room system at about (x, y)
*/
void caosVM::c_MVSF() {
	// TODO
	c_MVTO();
}

/**
 FRIC (float)
*/
void caosVM::v_FRIC() {
	VM_VERIFY_SIZE(0)

	assert(targ);
	result.setFloat(targ->friction);
}

/**
 FRIC (command) friction (integer)
*/
void caosVM::c_FRIC() {
	VM_VERIFY_SIZE(1)
	VM_PARAM_INTEGER(friction) caos_assert(friction >= 0); caos_assert(friction <= 100);

	assert(targ);
	targ->friction = friction;
}

/**
 FALL (integer)

 Return 1 if TARG is moving due to gravity, 0 otherwise
*/
void caosVM::v_FALL() {
	VM_VERIFY_SIZE(0)
	caos_assert(targ);

	float accg = targ->accg;
	float vely = targ->vely.getFloat();
	if (accg != 0 && vely != 0) {
		// XXX: this probably isn't quite correct, but it's close enough for
		// now. Hook it into the physics system properly later
		result.setInt(1);
	} else
		result.setInt(0);
}
<|endoftext|>


<commit_before>#pragma once

#include <iostream>
#include <stdexcept>
#include <vector>
#include <map>
#include <stack>
#include <typeinfo>
#include <memory>

#include "rapidjson/reader.h"
#include "rapidjson/error/en.h"
#include "rapidjson/stringbuffer.h"

#include "acmacs-base/read-file.hh"

// ----------------------------------------------------------------------

namespace json_reader
{
    class Error : public std::runtime_error { public: using std::runtime_error::runtime_error; };

    class Failure : public std::runtime_error { public: using std::runtime_error::runtime_error; inline Failure() : std::runtime_error{""} {} };
    class Pop : public std::exception { public: using std::exception::exception; };

// ----------------------------------------------------------------------

    template <typename Target> class HandlerBase
    {
     public:
        inline HandlerBase(Target& aTarget) : mTarget(aTarget), mIgnore(false) {}
        virtual ~HandlerBase() {}

        inline virtual HandlerBase* StartObject() { throw Failure(std::string("HandlerBase StartObject ") + typeid(*this).name()); }
        inline virtual HandlerBase* EndObject() { throw Pop(); }
        inline virtual HandlerBase* StartArray() { throw Failure(std::string("HandlerBase StartArray ") + typeid(*this).name()); }
        inline virtual HandlerBase* EndArray() { throw Pop(); }
        inline virtual HandlerBase* Double(double d) { if (mIgnore) { mIgnore = false; return nullptr; } throw Failure("HandlerBase Double " + std::to_string(d)); }
        inline virtual HandlerBase* Int(int i) { if (mIgnore) { mIgnore = false; return nullptr; } throw Failure("HandlerBase Int " + std::to_string(i)); }
        inline virtual HandlerBase* Uint(unsigned u) { if (mIgnore) { mIgnore = false; return nullptr; } throw Failure("HandlerBase Uint " + std::to_string(u)); }
        inline virtual HandlerBase* Bool(bool b) { if (mIgnore) { mIgnore = false; return nullptr; } throw Failure("HandlerBase Bool " + std::to_string(b)); }

        inline virtual HandlerBase* Key(const char* str, rapidjson::SizeType length)
            {
                if ((length == 1 && *str == '_') || (length > 0 && (*str == '?' || str[length - 1] == '?')))
                    mIgnore = true;
                else
                    throw Failure("HandlerBase Key: \"" + std::string(str, length) + "\"");
                return nullptr;
            }

        inline virtual HandlerBase* String(const char* str, rapidjson::SizeType length)
            {
                if (mIgnore)
                    mIgnore = false;
                else
                    throw Failure("HandlerBase String: \"" + std::string(str, length) + "\"");
                return nullptr;
            }

     protected:
        Target& mTarget;
        bool mIgnore;
    };

// ----------------------------------------------------------------------

    template <typename Target, typename Element, typename ElementHandler> class ListHandler : public HandlerBase<Target>
    {
     public:
        inline ListHandler(Target& aTarget, std::vector<Element>& aList)
            : HandlerBase<Target>(aTarget), mList(aList), mStarted(false) {}

        inline virtual HandlerBase<Target>* StartArray()
            {
                if (mStarted)
                    throw Failure{};
                mStarted = true;
                return nullptr;
            }

        inline virtual HandlerBase<Target>* StartObject()
            {
                if (!mStarted)
                    throw Failure{};
                mList.emplace_back();
                return new ElementHandler(HandlerBase<Target>::mTarget, mList.back());
            }

        inline virtual HandlerBase<Target>* EndObject() { throw Failure(); }

     private:
        std::vector<Element>& mList;
        bool mStarted;

    }; // class StringListHandler

// ----------------------------------------------------------------------

    template <typename Target> class StringListHandler : public HandlerBase<Target>
    {
     public:
        inline StringListHandler(Target& aTarget, std::vector<std::string>& aList)
            : HandlerBase<Target>(aTarget), mList(aList), mStarted(false) {}

        inline virtual HandlerBase<Target>* StartArray()
            {
                if (mStarted)
                    throw Failure();
                mStarted = true;
                return nullptr;
            }

        inline virtual HandlerBase<Target>* EndObject() { throw Failure(); }

        inline virtual HandlerBase<Target>* String(const char* str, rapidjson::SizeType length)
            {
                mList.emplace_back(str, length);
                return nullptr;
            }

     private:
        std::vector<std::string>& mList;
        bool mStarted;

    }; // class StringListHandler

// ----------------------------------------------------------------------

    template <typename Target> class MapListHandler : public HandlerBase<Target>
    {
     public:
        inline MapListHandler(Target& aTarget, std::map<std::string, std::vector<std::string>>& aMap)
            : HandlerBase<Target>{aTarget}, mMap(aMap), mStarted(false) {}

        inline virtual HandlerBase<Target>* StartObject()
            {
                if (mStarted)
                    throw Failure();
                mStarted = true;
                return nullptr;
            }

        inline virtual HandlerBase<Target>* EndArray() { throw Failure(); }

        inline virtual HandlerBase<Target>* Key(const char* str, rapidjson::SizeType length)
            {
                return new StringListHandler<Target>(HandlerBase<Target>::mTarget, mMap[{str, length}]);
            }

     private:
        std::map<std::string, std::vector<std::string>>& mMap;
        bool mStarted;

    }; // class MapListHandler

// ----------------------------------------------------------------------

    template <typename Target> class StringMappingHandler : public HandlerBase<Target>
    {
     public:
        inline StringMappingHandler(Target& aTarget, std::vector<std::pair<std::string, std::string>>& aMapping)
            : HandlerBase<Target>{aTarget}, mMapping(aMapping), mStarted(false) {}

        inline virtual HandlerBase<Target>* StartObject()
            {
                if (mStarted)
                    throw json_reader::Failure();
                mStarted = true;
                return nullptr;
            }

        inline virtual HandlerBase<Target>* Key(const char* str, rapidjson::SizeType length)
            {
                mKey.assign(str, length);
                return nullptr;
            }

        inline virtual HandlerBase<Target>* String(const char* str, rapidjson::SizeType length)
            {
                if (mKey.empty())
                    throw json_reader::Failure();
                mMapping.emplace_back(mKey, std::string(str, length));
                mKey.erase();
                return nullptr;
            }

     private:
        std::vector<std::pair<std::string, std::string>>& mMapping;
        bool mStarted;
        std::string mKey;

    }; // class StringMappingHandler

// ----------------------------------------------------------------------

    template <typename Target, typename RootHandler> class DocRootHandler : public HandlerBase<Target>
    {
     public:
        inline DocRootHandler(Target& aTarget) : HandlerBase<Target>(aTarget) {}

        inline virtual HandlerBase<Target>* StartObject() { return new RootHandler(HandlerBase<Target>::mTarget); }
    };

// ----------------------------------------------------------------------

    template <typename Target, typename RootHandler> class ReaderEventHandler : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>, ReaderEventHandler<Target, RootHandler>>
    {
     private:
        template <typename... Args> inline bool handler(HandlerBase<Target>* (HandlerBase<Target>::*aHandler)(Args... args), Args... args)
            {
                try {
                    auto new_handler = ((*mHandler.top()).*aHandler)(args...);
                    if (new_handler)
                        mHandler.emplace(new_handler);
                }
                catch (Pop&) {
                    if (mHandler.empty())
                        return false;
                    mHandler.pop();
                }
                catch (Failure& err) {
                    if (*err.what())
                        std::cerr << "ERROR: " << err.what() << std::endl;
                    return false;
                }
                return true;
            }

     public:
        inline ReaderEventHandler(Target& aTarget)
            : mTarget(aTarget)
            {
                mHandler.emplace(new DocRootHandler<Target, RootHandler>(mTarget));
            }

        inline bool StartObject() { return handler(&HandlerBase<Target>::StartObject); }
        inline bool EndObject(rapidjson::SizeType /*memberCount*/) { return handler(&HandlerBase<Target>::EndObject); }
        inline bool StartArray() { return handler(&HandlerBase<Target>::StartArray); }
        inline bool EndArray(rapidjson::SizeType /*elementCount*/) { return handler(&HandlerBase<Target>::EndArray); }
        inline bool Key(const char* str, rapidjson::SizeType length, bool /*copy*/) { return handler(&HandlerBase<Target>::Key, str, length); }
        inline bool String(const char* str, rapidjson::SizeType length, bool /*copy*/) { return handler(&HandlerBase<Target>::String, str, length); }
        inline bool Int(int i) { return handler(&HandlerBase<Target>::Int, i); }
        inline bool Uint(unsigned u) { return handler(&HandlerBase<Target>::Uint, u); }
        inline bool Double(double d) { return handler(&HandlerBase<Target>::Double, d); }
        inline bool Bool(bool b) { return handler(&HandlerBase<Target>::Bool, b); }
        inline bool Null() { std::cerr << "ReaderEventHandler::Null()" << std::endl; return false; }
        inline bool Int64(int64_t i) { std::cerr << "ReaderEventHandler::Int64(" << i << ")" << std::endl; return false; }
        inline bool Uint64(uint64_t u) { std::cerr << "ReaderEventHandler::Uint64(" << u << ")" << std::endl; return false; }

     private:
        Target& mTarget;
        std::stack<std::unique_ptr<HandlerBase<Target>>> mHandler;
    };

// ----------------------------------------------------------------------

    template <typename Target, typename RootHandler> inline void read_from_file(std::string aFilename, Target& aTarget)
    {
        const std::string buffer = aFilename == "-" ? acmacs_base::read_stdin() : acmacs_base::read_file(aFilename);
        if (buffer[0] == '{') {
            ReaderEventHandler<Target, RootHandler> handler{aTarget};
            rapidjson::Reader reader;
            rapidjson::StringStream stream{buffer.c_str()};
            reader.Parse(stream, handler);
            if (reader.HasParseError())
                throw Error("cannot read " + aFilename + ": data parsing failed at pos " + std::to_string(reader.GetErrorOffset()) + ": " +  GetParseError_En(reader.GetParseErrorCode()) + "\n" + buffer.substr(reader.GetErrorOffset(), 50));
        }
        else
            throw json_reader::Error("cannot read " + aFilename + ": unrecognized source format");
    }

// ----------------------------------------------------------------------

} // namespace json_reader

// ----------------------------------------------------------------------
/// Local Variables:
/// eval: (if (fboundp 'eu-rename-buffer) (eu-rename-buffer))
/// End:
<commit_msg>json parser support improvements<commit_after>#pragma once

#include <iostream>
#include <stdexcept>
#include <vector>
#include <map>
#include <stack>
#include <typeinfo>
#include <memory>

#include "rapidjson/reader.h"
#include "rapidjson/error/en.h"
#include "rapidjson/stringbuffer.h"

#include "acmacs-base/read-file.hh"

// ----------------------------------------------------------------------

namespace json_reader
{
    class Error : public std::runtime_error { public: using std::runtime_error::runtime_error; };

    class Failure : public std::runtime_error { public: using std::runtime_error::runtime_error; inline Failure() : std::runtime_error{""} {} };
    class Pop : public std::exception { public: using std::exception::exception; };

// ----------------------------------------------------------------------

    template <typename Target> class HandlerBase
    {
     public:
        inline HandlerBase(Target& aTarget) : mTarget(aTarget), mIgnore(false) {}
        virtual ~HandlerBase() {}

        inline virtual HandlerBase* StartObject() { throw Failure(std::string("HandlerBase StartObject ") + typeid(*this).name()); }
        inline virtual HandlerBase* EndObject() { throw Pop(); }
        inline virtual HandlerBase* StartArray() { throw Failure(std::string("HandlerBase StartArray ") + typeid(*this).name()); }
        inline virtual HandlerBase* EndArray() { throw Pop(); }
        inline virtual HandlerBase* Double(double d) { if (mIgnore) { mIgnore = false; return nullptr; } throw Failure("HandlerBase Double " + std::to_string(d)); }
        inline virtual HandlerBase* Int(int i) { if (mIgnore) { mIgnore = false; return nullptr; } throw Failure("HandlerBase Int " + std::to_string(i)); }
        inline virtual HandlerBase* Uint(unsigned u) { if (mIgnore) { mIgnore = false; return nullptr; } throw Failure("HandlerBase Uint " + std::to_string(u)); }
        inline virtual HandlerBase* Bool(bool b) { if (mIgnore) { mIgnore = false; return nullptr; } throw Failure("HandlerBase Bool " + std::to_string(b)); }
        inline virtual HandlerBase* Null() { if (mIgnore) { mIgnore = false; return nullptr; } throw Failure("HandlerBase Null"); }

        inline virtual HandlerBase* Key(const char* str, rapidjson::SizeType length)
            {
                if ((length == 1 && *str == '_') || (length > 0 && (*str == '?' || str[length - 1] == '?')))
                    mIgnore = true;
                else
                    throw Failure("HandlerBase Key: \"" + std::string(str, length) + "\"");
                return nullptr;
            }

        inline virtual HandlerBase* String(const char* str, rapidjson::SizeType length)
            {
                if (mIgnore)
                    mIgnore = false;
                else
                    throw Failure("HandlerBase String: \"" + std::string(str, length) + "\"");
                return nullptr;
            }

     protected:
        Target& mTarget;
        bool mIgnore;
    };

// ----------------------------------------------------------------------

    template <typename Target> class GenericListHandler : public HandlerBase<Target>
    {
     public:
        inline GenericListHandler(Target& aTarget)
            : HandlerBase<Target>(aTarget), mStarted(false) {}

        inline virtual HandlerBase<Target>* StartArray()
            {
                if (mStarted)
                    throw Failure{};
                mStarted = true;
                return nullptr;
            }

        inline virtual HandlerBase<Target>* EndObject() { throw Failure(); }

     protected:
        inline bool started() const { return mStarted; }

     private:
        bool mStarted;

    }; // class GenericListHandler

// ----------------------------------------------------------------------

    template <typename Target, typename Element, typename ElementHandler> class ListHandler : public GenericListHandler<Target>
    {
     public:
        inline ListHandler(Target& aTarget, std::vector<Element>& aList)
            : GenericListHandler<Target>(aTarget), mList(aList) {}

        inline virtual HandlerBase<Target>* StartObject()
            {
                if (!this->started())
                    throw Failure{};
                mList.emplace_back();
                return new ElementHandler(HandlerBase<Target>::mTarget, mList.back());
            }

     private:
        std::vector<Element>& mList;

    }; // class ListHandler

// ----------------------------------------------------------------------

    template <typename Target> class StringListHandler : public GenericListHandler<Target>
    {
     public:
        inline StringListHandler(Target& aTarget, std::vector<std::string>& aList)
            : GenericListHandler<Target>(aTarget), mList(aList) {}

        inline virtual HandlerBase<Target>* String(const char* str, rapidjson::SizeType length)
            {
                mList.emplace_back(str, length);
                return nullptr;
            }

     private:
        std::vector<std::string>& mList;

    }; // class StringListHandler

// ----------------------------------------------------------------------

    template <typename Target> class UintListHandler : public GenericListHandler<Target>
    {
     public:
        inline UintListHandler(Target& aTarget, std::vector<size_t>& aList)
            : GenericListHandler<Target>(aTarget), mList(aList) {}

        inline virtual HandlerBase<Target>* Uint(unsigned u)
            {
                mList.push_back(u);
                return nullptr;
            }

     private:
        std::vector<size_t>& mList;

    }; // class UintListHandler

// ----------------------------------------------------------------------

    template <typename Target> class MapListHandler : public HandlerBase<Target>
    {
     public:
        inline MapListHandler(Target& aTarget, std::map<std::string, std::vector<std::string>>& aMap)
            : HandlerBase<Target>{aTarget}, mMap(aMap), mStarted(false) {}

        inline virtual HandlerBase<Target>* StartObject()
            {
                if (mStarted)
                    throw Failure();
                mStarted = true;
                return nullptr;
            }

        inline virtual HandlerBase<Target>* EndArray() { throw Failure(); }

        inline virtual HandlerBase<Target>* Key(const char* str, rapidjson::SizeType length)
            {
                return new StringListHandler<Target>(HandlerBase<Target>::mTarget, mMap[{str, length}]);
            }

     private:
        std::map<std::string, std::vector<std::string>>& mMap;
        bool mStarted;

    }; // class MapListHandler

// ----------------------------------------------------------------------

    template <typename Target> class StringMappingHandler : public HandlerBase<Target>
    {
     public:
        inline StringMappingHandler(Target& aTarget, std::vector<std::pair<std::string, std::string>>& aMapping)
            : HandlerBase<Target>{aTarget}, mMapping(aMapping), mStarted(false) {}

        inline virtual HandlerBase<Target>* StartObject()
            {
                if (mStarted)
                    throw json_reader::Failure();
                mStarted = true;
                return nullptr;
            }

        inline virtual HandlerBase<Target>* Key(const char* str, rapidjson::SizeType length)
            {
                mKey.assign(str, length);
                return nullptr;
            }

        inline virtual HandlerBase<Target>* String(const char* str, rapidjson::SizeType length)
            {
                if (mKey.empty())
                    throw json_reader::Failure();
                mMapping.emplace_back(mKey, std::string(str, length));
                mKey.erase();
                return nullptr;
            }

     private:
        std::vector<std::pair<std::string, std::string>>& mMapping;
        bool mStarted;
        std::string mKey;

    }; // class StringMappingHandler

// ----------------------------------------------------------------------

    template <typename Target, typename RootHandler> class DocRootHandler : public HandlerBase<Target>
    {
     public:
        inline DocRootHandler(Target& aTarget) : HandlerBase<Target>(aTarget) {}

        inline virtual HandlerBase<Target>* StartObject() { return new RootHandler(HandlerBase<Target>::mTarget); }
    };

// ----------------------------------------------------------------------

    template <typename Target, typename RootHandler> class ReaderEventHandler : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>, ReaderEventHandler<Target, RootHandler>>
    {
     private:
        template <typename... Args> inline bool handler(HandlerBase<Target>* (HandlerBase<Target>::*aHandler)(Args... args), Args... args)
            {
                try {
                    auto new_handler = ((*mHandler.top()).*aHandler)(args...);
                    if (new_handler)
                        mHandler.emplace(new_handler);
                }
                catch (Pop&) {
                    if (mHandler.empty())
                        return false;
                    mHandler.pop();
                }
                catch (Failure& err) {
                    if (*err.what())
                        std::cerr << "ERROR: " << err.what() << std::endl;
                    return false;
                }
                return true;
            }

     public:
        inline ReaderEventHandler(Target& aTarget)
            : mTarget(aTarget)
            {
                mHandler.emplace(new DocRootHandler<Target, RootHandler>(mTarget));
            }

        inline bool StartObject() { return handler(&HandlerBase<Target>::StartObject); }
        inline bool EndObject(rapidjson::SizeType /*memberCount*/) { return handler(&HandlerBase<Target>::EndObject); }
        inline bool StartArray() { return handler(&HandlerBase<Target>::StartArray); }
        inline bool EndArray(rapidjson::SizeType /*elementCount*/) { return handler(&HandlerBase<Target>::EndArray); }
        inline bool Key(const char* str, rapidjson::SizeType length, bool /*copy*/) { return handler(&HandlerBase<Target>::Key, str, length); }
        inline bool String(const char* str, rapidjson::SizeType length, bool /*copy*/) { return handler(&HandlerBase<Target>::String, str, length); }
        inline bool Int(int i) { return handler(&HandlerBase<Target>::Int, i); }
        inline bool Uint(unsigned u) { return handler(&HandlerBase<Target>::Uint, u); }
        inline bool Double(double d) { return handler(&HandlerBase<Target>::Double, d); }
        inline bool Bool(bool b) { return handler(&HandlerBase<Target>::Bool, b); }
        inline bool Null() { return handler(&HandlerBase<Target>::Null); }
        inline bool Int64(int64_t i) { std::cerr << "ReaderEventHandler::Int64(" << i << ")" << std::endl; return false; }
        inline bool Uint64(uint64_t u) { std::cerr << "ReaderEventHandler::Uint64(" << u << ")" << std::endl; return false; }

     private:
        Target& mTarget;
        std::stack<std::unique_ptr<HandlerBase<Target>>> mHandler;
    };

// ----------------------------------------------------------------------

    template <typename Target, typename RootHandler> inline void read_from_file(std::string aFilename, Target& aTarget)
    {
        const std::string buffer = aFilename == "-" ? acmacs_base::read_stdin() : acmacs_base::read_file(aFilename);
        if (buffer[0] == '{') {
            ReaderEventHandler<Target, RootHandler> handler{aTarget};
            rapidjson::Reader reader;
            rapidjson::StringStream stream{buffer.c_str()};
            reader.Parse(stream, handler);
            if (reader.HasParseError())
                throw Error("cannot read " + aFilename + ": data parsing failed at pos " + std::to_string(reader.GetErrorOffset()) + ": " +  GetParseError_En(reader.GetParseErrorCode()) + "\n" + buffer.substr(reader.GetErrorOffset(), 50));
        }
        else
            throw json_reader::Error("cannot read " + aFilename + ": unrecognized source format");
    }

// ----------------------------------------------------------------------

} // namespace json_reader

// ----------------------------------------------------------------------
/// Local Variables:
/// eval: (if (fboundp 'eu-rename-buffer) (eu-rename-buffer))
/// End:
<|endoftext|>


<commit_before>#pragma once

#include <string>
#include <vector>
#include <map>
#include <set>

#include "acmacs-base/config.hh"

#pragma GCC diagnostic push
#include "acmacs-base/rapidjson-diagnostics.hh"
#include "rapidjson/writer.h"
#include "rapidjson/prettywriter.h"
#include "rapidjson/error/en.h"
#include "rapidjson/stringbuffer.h"
#pragma GCC diagnostic pop

#ifdef ACMACSD_FILESYSTEM
#include "acmacs-base/read-file.hh"
#endif
#include "acmacs-base/float.hh"
#include "acmacs-base/sfinae.hh"

// ----------------------------------------------------------------------
// Forward declarations
// ----------------------------------------------------------------------

namespace json_writer
{
    template <typename RW> class writer;
    class key;
}

template <typename RW, typename T> json_writer::writer<RW>& operator <<(json_writer::writer<RW>&, const std::vector<T>&);
template <typename RW> json_writer::writer<RW>& operator <<(json_writer::writer<RW>&, const char*);
template <typename RW> json_writer::writer<RW>& operator <<(json_writer::writer<RW>&, std::string);
template <typename RW> json_writer::writer<RW>& operator <<(json_writer::writer<RW>&, double);
template <typename RW> json_writer::writer<RW>& operator <<(json_writer::writer<RW>&, int);
template <typename RW> json_writer::writer<RW>& operator <<(json_writer::writer<RW>&, size_t);
template <typename RW> json_writer::writer<RW>& operator <<(json_writer::writer<RW>&, bool);
template <typename RW, typename Value> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, const std::vector<Value>& aList);
template <typename RW, typename Value> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, const std::set<Value>& aList);
template <typename RW, typename Value> json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, const std::map<std::string, Value>& map_value);
template <typename RW> json_writer::writer<RW>& operator <<(json_writer::writer<RW>&, json_writer::key);

// ======================================================================

namespace json_writer
{
    template <typename RW> class writer : public RW // JsonWriterT
    {
     public:
        inline writer(std::string aKeyword) : RW(mBuffer), mKeyword(aKeyword) {}
        inline std::string to_string() const { return mBuffer.GetString(); }
        inline std::string keyword() const { return mKeyword; }

     private:
        rapidjson::StringBuffer mBuffer;
        std::string mKeyword;
    };

      // template <> inline writer<rapidjson::PrettyWriter<rapidjson::StringBuffer>>::writer(std::string aKeyword)
      //     : rapidjson::PrettyWriter<rapidjson::StringBuffer>(mBuffer), mKeyword(aKeyword)
      // {
      //     SetIndent(' ', 1);
      // }

      // ----------------------------------------------------------------------

    enum _StartArray { start_array };
    enum _EndArray { end_array };
    enum _StartObject { start_object };
    enum _EndObject { end_object };

    class key
    {
     public:
        inline key(const char* v) : value(v) {}
        inline key(std::string v) : value(v) {}
        inline key(char v) : value(1, v) {}
        inline operator const char*() const { return value.c_str(); }
        inline const char* as_char_ptr() const { return value.c_str(); }

     private:
        std::string value;
    };
}

// ----------------------------------------------------------------------

template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, json_writer::_StartArray) { aWriter.StartArray(); return aWriter; }
template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, json_writer::_EndArray) { aWriter.EndArray(); return aWriter; }
template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, json_writer::_StartObject) { aWriter.StartObject(); return aWriter; }
template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, json_writer::_EndObject) { aWriter.EndObject(); return aWriter; }

template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, json_writer::key value) { aWriter.Key(value.as_char_ptr()); return aWriter; }

// ----------------------------------------------------------------------
// ----------------------------------------------------------------------

namespace json_writer
{
    template <typename Key, typename Value> class _if_not_empty
    {
     public:
        inline _if_not_empty(Key&& aKey, Value&& aValue) : mKey(aKey), mValue(aValue) {}

        template <typename RW> friend inline writer<RW>& operator <<(writer<RW>& aWriter, const _if_not_empty<Key, Value>& data)
            {
                if (!data.mValue.empty())
                    aWriter << data.mKey << data.mValue;
                return aWriter;
            }

     private:
        Key mKey;
        Value mValue;
    };

    template <typename Key, typename Value> inline auto if_not_empty(Key&& aKey, Value&& aValue) { return _if_not_empty<Key, Value>(std::forward<Key>(aKey), std::forward<Value>(aValue)); }
    template <typename Value> inline auto if_not_empty(const char* aKey, Value&& aValue) { return _if_not_empty<key, Value>(key(aKey), std::forward<Value>(aValue)); }

// ----------------------------------------------------------------------

    template <typename Key, typename Value> class _if_non_negative
    {
     public:
        inline _if_non_negative(Key&& aKey, Value&& aValue) : mKey(aKey), mValue(aValue) {}

        template <typename RW> friend inline writer<RW>& operator <<(writer<RW>& aWriter, const _if_non_negative<Key, Value>& data)
            {
                if (data.mValue >= 0.0)
                    aWriter << data.mKey << data.mValue;
                return aWriter;
            }

     private:
        Key mKey;
        Value mValue;
    };

    template <typename Key, typename Value> inline auto if_non_negative(Key&& aKey, Value&& aValue) { return _if_non_negative<Key, Value>(std::forward<Key>(aKey), std::forward<Value>(aValue)); }
    template <typename Value> inline auto if_non_negative(const char* aKey, Value&& aValue) { return _if_non_negative<key, Value>(key(aKey), std::forward<Value>(aValue)); }

// ----------------------------------------------------------------------

    template <typename Key, typename Value> class _if_not
    {
     public:
        inline _if_not(Key&& aKey, Value&& aValue, bool aPred) : mKey(aKey), mValue(aValue), mPred(aPred) {}
        inline _if_not(Key&& aKey, Value aValue, bool aPred) : mKey(aKey), mValue(aValue), mPred(aPred) {}

        template <typename RW> friend inline writer<RW>& operator <<(writer<RW>& aWriter, const _if_not<Key, Value>& data)
            {
                if (!data.mPred)
                    aWriter << data.mKey << data.mValue;
                return aWriter;
            }

     private:
        Key mKey;
        Value mValue;
        bool mPred;
    };

    template <typename Key> inline auto if_not_zero(Key&& aKey, double aValue) { return _if_not<Key, double>(std::forward<Key>(aKey), aValue, float_zero(aValue)); }
    inline auto if_not_zero(const char* aKey, double aValue) { return _if_not<key, double>(key(aKey), aValue, float_zero(aValue)); }

    template <typename Key> inline auto if_not_one(Key&& aKey, double aValue) { return _if_not<Key, double>(std::forward<Key>(aKey), aValue, float_equal(aValue, 1.0)); }
    inline auto if_not_one(const char* aKey, double aValue) { return _if_not<key, double>(key(aKey), aValue, float_equal(aValue, 1.0)); }

    template <typename Key, typename Value> inline auto if_not_equal(Key&& aKey, Value&& aValue, Value aEq) { return _if_not<Key, Value>(std::forward<Key>(aKey), aValue, aValue == aEq); }
    template <typename Value> inline auto if_not_equal(const char* aKey, Value&& aValue, Value aEq) { return _if_not<key, Value>(key(aKey), aValue, aValue == aEq); }

    template <typename RW, typename List> inline writer<RW>& write_list(writer<RW>& aWriter, const List& aList)
    {
          // start_array and end_array do not work here due to namespace issue (in clang only?)
        aWriter.StartArray();
        for (const auto& e: aList)
            aWriter << e;
        aWriter.EndArray();
        return aWriter;
    }

    template <typename RW, typename List> inline writer<RW>& write_list(writer<RW>& aWriter, List&& aList)
    {
          // start_array and end_array do not work here due to namespace issue (in clang only?)
        aWriter.StartArray();
        for (auto& e: aList)
            aWriter << e;
        aWriter.EndArray();
        return aWriter;
    }

    template <typename RW, typename Object> inline writer<RW>& write_object(writer<RW>& aWriter, const Object& aObject)
    {
          // start_object and end_object do not work here due to namespace issue (in clang only?)
        aWriter.StartObject();
        for (const auto& e: aObject)
            aWriter << json_writer::key(e.first) << e.second;
        aWriter.EndObject();
        return aWriter;
    }

} // namespace json_writer

// ----------------------------------------------------------------------
// ----------------------------------------------------------------------

// to allow writer << func(writer, data) where func returns writer
template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, json_writer::writer<RW>&) { return aWriter; }

template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, const char* s) { aWriter.String(s, static_cast<unsigned>(strlen(s))); return aWriter; }
template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, std::string s) { aWriter.String(s.c_str(), static_cast<unsigned>(s.size())); return aWriter; }
template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, int value) { aWriter.Int(value); return aWriter; }
template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, size_t value) { aWriter.Uint64(value); return aWriter; }
template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, bool value) { aWriter.Bool(value); return aWriter; }
template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, double value) { aWriter.Double(value); return aWriter; }

template <typename RW, typename Value> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, const std::vector<Value>& aList)
{
    return json_writer::write_list(aWriter, aList);
}

template <typename RW, typename Value> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, const std::set<Value>& aList)
{
    return json_writer::write_list(aWriter, aList);
}

template <typename RW, typename Value> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, const std::map<std::string, Value>& map_value)
{
    return json_writer::write_object(aWriter, map_value);
}

template <typename RW, typename Key, typename std::enable_if<ad_sfinae::castable_to_char<Key>{}>::type* = nullptr>
                      inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, Key aKey)
{
    const char k = static_cast<char>(aKey);
    aWriter.Key(&k, 1, false);
    return aWriter;
}

// ----------------------------------------------------------------------
// ----------------------------------------------------------------------

namespace json_writer
{
    enum _Finalize { finalize };

    class pretty;
}

std::string operator << (json_writer::pretty& aWriter, json_writer::_Finalize);

namespace json_writer
{
    inline void insert_emacs_indent_hint(std::string& aJson, size_t aIndent)
    {
        if (aJson.size() > 2 && aJson[0] == '{') {
            const std::string ind(aIndent - 1, ' ');
            aJson.insert(1, ind + "\"_\": \"-*- js-indent-level: " + std::to_string(aIndent) + " -*-\",");
        }
    }

    using compact = writer<rapidjson::Writer<rapidjson::StringBuffer>>;

    class pretty : public writer<rapidjson::PrettyWriter<rapidjson::StringBuffer>>
    {
     public:
        inline pretty(std::string keyword, size_t indent = 1) : writer<rapidjson::PrettyWriter<rapidjson::StringBuffer>>{keyword}, mIndent{indent}
            {
                SetIndent(' ', static_cast<unsigned int>(indent));
            }

        inline size_t indent() const { return mIndent; }
        inline operator std::string() { return *this << finalize; }

     private:
        size_t mIndent;

    }; // class pretty

} // namespace json_writer

// ----------------------------------------------------------------------

inline std::string operator << (json_writer::compact& aWriter, json_writer::_Finalize)
{
    return aWriter.to_string();
}

inline std::string operator << (json_writer::pretty& aWriter, json_writer::_Finalize)
{
    std::string result = aWriter.to_string();
    json_writer::insert_emacs_indent_hint(result, aWriter.indent());
    return result;
}

// ----------------------------------------------------------------------

namespace json_writer
{
    template <typename V> inline std::string pretty_json(const V& value, std::string keyword, size_t indent)
    {
        pretty aWriter{keyword, indent};
        return aWriter << value << finalize;
    }

    template <typename V> inline std::string compact_json(const V& aValue, std::string keyword)
    {
        compact aWriter{keyword};
        return aWriter << aValue;
    }

    template <typename V> inline std::string json(const V& value, std::string keyword, size_t indent)
    {
        if (indent)
            return pretty_json(value, keyword, indent);
        else
            return compact_json(value, keyword);
    }

      // ----------------------------------------------------------------------

#ifdef ACMACSD_FILESYSTEM
    template <typename V> inline void export_to_json(const V& value, std::string keyword, std::string filename, size_t indent, bool insert_emacs_indent_hint = true, bool force_compression = false)
    {
        acmacs_base::write_file(filename, json(value, keyword, indent, insert_emacs_indent_hint), force_compression);
    }
#endif

} // namespace json_writer

// ----------------------------------------------------------------------
/// Local Variables:
/// eval: (if (fboundp 'eu-rename-buffer) (eu-rename-buffer))
/// End:
<commit_msg>bug fix<commit_after>#pragma once

#include <string>
#include <vector>
#include <map>
#include <set>

#include "acmacs-base/config.hh"

#pragma GCC diagnostic push
#include "acmacs-base/rapidjson-diagnostics.hh"
#include "rapidjson/writer.h"
#include "rapidjson/prettywriter.h"
#include "rapidjson/error/en.h"
#include "rapidjson/stringbuffer.h"
#pragma GCC diagnostic pop

#ifdef ACMACSD_FILESYSTEM
#include "acmacs-base/read-file.hh"
#endif
#include "acmacs-base/float.hh"
#include "acmacs-base/sfinae.hh"

// ----------------------------------------------------------------------
// Forward declarations
// ----------------------------------------------------------------------

namespace json_writer
{
    template <typename RW> class writer;
    class key;
}

template <typename RW, typename T> json_writer::writer<RW>& operator <<(json_writer::writer<RW>&, const std::vector<T>&);
template <typename RW> json_writer::writer<RW>& operator <<(json_writer::writer<RW>&, const char*);
template <typename RW> json_writer::writer<RW>& operator <<(json_writer::writer<RW>&, std::string);
template <typename RW> json_writer::writer<RW>& operator <<(json_writer::writer<RW>&, double);
template <typename RW> json_writer::writer<RW>& operator <<(json_writer::writer<RW>&, int);
template <typename RW> json_writer::writer<RW>& operator <<(json_writer::writer<RW>&, size_t);
template <typename RW> json_writer::writer<RW>& operator <<(json_writer::writer<RW>&, bool);
template <typename RW, typename Value> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, const std::vector<Value>& aList);
template <typename RW, typename Value> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, const std::set<Value>& aList);
template <typename RW, typename Value> json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, const std::map<std::string, Value>& map_value);
template <typename RW> json_writer::writer<RW>& operator <<(json_writer::writer<RW>&, json_writer::key);

// ======================================================================

namespace json_writer
{
    template <typename RW> class writer : public RW // JsonWriterT
    {
     public:
        inline writer(std::string aKeyword) : RW(mBuffer), mKeyword(aKeyword) {}
        inline std::string to_string() const { return mBuffer.GetString(); }
        inline std::string keyword() const { return mKeyword; }

     private:
        rapidjson::StringBuffer mBuffer;
        std::string mKeyword;
    };

      // template <> inline writer<rapidjson::PrettyWriter<rapidjson::StringBuffer>>::writer(std::string aKeyword)
      //     : rapidjson::PrettyWriter<rapidjson::StringBuffer>(mBuffer), mKeyword(aKeyword)
      // {
      //     SetIndent(' ', 1);
      // }

      // ----------------------------------------------------------------------

    enum _StartArray { start_array };
    enum _EndArray { end_array };
    enum _StartObject { start_object };
    enum _EndObject { end_object };

    class key
    {
     public:
        inline key(const char* v) : value(v) {}
        inline key(std::string v) : value(v) {}
        inline key(char v) : value(1, v) {}
        inline operator const char*() const { return value.c_str(); }
        inline const char* as_char_ptr() const { return value.c_str(); }

     private:
        std::string value;
    };
}

// ----------------------------------------------------------------------

template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, json_writer::_StartArray) { aWriter.StartArray(); return aWriter; }
template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, json_writer::_EndArray) { aWriter.EndArray(); return aWriter; }
template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, json_writer::_StartObject) { aWriter.StartObject(); return aWriter; }
template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, json_writer::_EndObject) { aWriter.EndObject(); return aWriter; }

template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, json_writer::key value) { aWriter.Key(value.as_char_ptr()); return aWriter; }

// ----------------------------------------------------------------------
// ----------------------------------------------------------------------

namespace json_writer
{
    template <typename Key, typename Value> class _if_not_empty
    {
     public:
        inline _if_not_empty(Key&& aKey, Value&& aValue) : mKey(aKey), mValue(aValue) {}

        template <typename RW> friend inline writer<RW>& operator <<(writer<RW>& aWriter, const _if_not_empty<Key, Value>& data)
            {
                if (!data.mValue.empty())
                    aWriter << data.mKey << data.mValue;
                return aWriter;
            }

     private:
        Key mKey;
        Value mValue;
    };

    template <typename Key, typename Value> inline auto if_not_empty(Key&& aKey, Value&& aValue) { return _if_not_empty<Key, Value>(std::forward<Key>(aKey), std::forward<Value>(aValue)); }
    template <typename Value> inline auto if_not_empty(const char* aKey, Value&& aValue) { return _if_not_empty<key, Value>(key(aKey), std::forward<Value>(aValue)); }

// ----------------------------------------------------------------------

    template <typename Key, typename Value> class _if_non_negative
    {
     public:
        inline _if_non_negative(Key&& aKey, Value&& aValue) : mKey(aKey), mValue(aValue) {}

        template <typename RW> friend inline writer<RW>& operator <<(writer<RW>& aWriter, const _if_non_negative<Key, Value>& data)
            {
                if (data.mValue >= 0.0)
                    aWriter << data.mKey << data.mValue;
                return aWriter;
            }

     private:
        Key mKey;
        Value mValue;
    };

    template <typename Key, typename Value> inline auto if_non_negative(Key&& aKey, Value&& aValue) { return _if_non_negative<Key, Value>(std::forward<Key>(aKey), std::forward<Value>(aValue)); }
    template <typename Value> inline auto if_non_negative(const char* aKey, Value&& aValue) { return _if_non_negative<key, Value>(key(aKey), std::forward<Value>(aValue)); }

// ----------------------------------------------------------------------

    template <typename Key, typename Value> class _if_not
    {
     public:
        inline _if_not(Key&& aKey, Value&& aValue, bool aPred) : mKey(aKey), mValue(aValue), mPred(aPred) {}
        inline _if_not(Key&& aKey, Value aValue, bool aPred) : mKey(aKey), mValue(aValue), mPred(aPred) {}

        template <typename RW> friend inline writer<RW>& operator <<(writer<RW>& aWriter, const _if_not<Key, Value>& data)
            {
                if (!data.mPred)
                    aWriter << data.mKey << data.mValue;
                return aWriter;
            }

     private:
        Key mKey;
        Value mValue;
        bool mPred;
    };

    template <typename Key> inline auto if_not_zero(Key&& aKey, double aValue) { return _if_not<Key, double>(std::forward<Key>(aKey), aValue, float_zero(aValue)); }
    inline auto if_not_zero(const char* aKey, double aValue) { return _if_not<key, double>(key(aKey), aValue, float_zero(aValue)); }

    template <typename Key> inline auto if_not_one(Key&& aKey, double aValue) { return _if_not<Key, double>(std::forward<Key>(aKey), aValue, float_equal(aValue, 1.0)); }
    inline auto if_not_one(const char* aKey, double aValue) { return _if_not<key, double>(key(aKey), aValue, float_equal(aValue, 1.0)); }

    template <typename Key, typename Value> inline auto if_not_equal(Key&& aKey, Value&& aValue, Value aEq) { return _if_not<Key, Value>(std::forward<Key>(aKey), aValue, aValue == aEq); }
    template <typename Value> inline auto if_not_equal(const char* aKey, Value&& aValue, Value aEq) { return _if_not<key, Value>(key(aKey), aValue, aValue == aEq); }

    template <typename RW, typename List> inline writer<RW>& write_list(writer<RW>& aWriter, const List& aList)
    {
          // start_array and end_array do not work here due to namespace issue (in clang only?)
        aWriter.StartArray();
        for (const auto& e: aList)
            aWriter << e;
        aWriter.EndArray();
        return aWriter;
    }

    template <typename RW, typename List> inline writer<RW>& write_list(writer<RW>& aWriter, List&& aList)
    {
          // start_array and end_array do not work here due to namespace issue (in clang only?)
        aWriter.StartArray();
        for (auto& e: aList)
            aWriter << e;
        aWriter.EndArray();
        return aWriter;
    }

    template <typename RW, typename Object> inline writer<RW>& write_object(writer<RW>& aWriter, const Object& aObject)
    {
          // start_object and end_object do not work here due to namespace issue (in clang only?)
        aWriter.StartObject();
        for (const auto& e: aObject)
            aWriter << json_writer::key(e.first) << e.second;
        aWriter.EndObject();
        return aWriter;
    }

} // namespace json_writer

// ----------------------------------------------------------------------
// ----------------------------------------------------------------------

// to allow writer << func(writer, data) where func returns writer
template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, json_writer::writer<RW>&) { return aWriter; }

template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, const char* s) { aWriter.String(s, static_cast<unsigned>(strlen(s))); return aWriter; }
template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, std::string s) { aWriter.String(s.c_str(), static_cast<unsigned>(s.size())); return aWriter; }
template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, int value) { aWriter.Int(value); return aWriter; }
template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, size_t value) { aWriter.Uint64(value); return aWriter; }
template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, bool value) { aWriter.Bool(value); return aWriter; }
template <typename RW> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, double value) { aWriter.Double(value); return aWriter; }

template <typename RW, typename Value> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, const std::vector<Value>& aList)
{
    return json_writer::write_list(aWriter, aList);
}

template <typename RW, typename Value> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, const std::set<Value>& aList)
{
    return json_writer::write_list(aWriter, aList);
}

template <typename RW, typename Value> inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, const std::map<std::string, Value>& map_value)
{
    return json_writer::write_object(aWriter, map_value);
}

template <typename RW, typename Key, typename std::enable_if<ad_sfinae::castable_to_char<Key>{}>::type* = nullptr>
                      inline json_writer::writer<RW>& operator <<(json_writer::writer<RW>& aWriter, Key aKey)
{
    const char k = static_cast<char>(aKey);
    aWriter.Key(&k, 1, false);
    return aWriter;
}

// ----------------------------------------------------------------------
// ----------------------------------------------------------------------

namespace json_writer
{
    enum _Finalize { finalize };

    class pretty;
}

std::string operator << (json_writer::pretty& aWriter, json_writer::_Finalize);

namespace json_writer
{
    inline void insert_emacs_indent_hint(std::string& aJson, size_t aIndent)
    {
        if (aJson.size() > 2 && aJson[0] == '{') {
            const std::string ind(aIndent - 1, ' ');
            aJson.insert(1, ind + "\"_\": \"-*- js-indent-level: " + std::to_string(aIndent) + " -*-\",");
        }
    }

    using compact = writer<rapidjson::Writer<rapidjson::StringBuffer>>;

    class pretty : public writer<rapidjson::PrettyWriter<rapidjson::StringBuffer>>
    {
     public:
        inline pretty(std::string keyword, size_t indent = 1) : writer<rapidjson::PrettyWriter<rapidjson::StringBuffer>>{keyword}, mIndent{indent}
            {
                SetIndent(' ', static_cast<unsigned int>(indent));
            }

        inline size_t indent() const { return mIndent; }
        inline operator std::string() { return *this << finalize; }

     private:
        size_t mIndent;

    }; // class pretty

} // namespace json_writer

// ----------------------------------------------------------------------

inline std::string operator << (json_writer::compact& aWriter, json_writer::_Finalize)
{
    return aWriter.to_string();
}

inline std::string operator << (json_writer::pretty& aWriter, json_writer::_Finalize)
{
    std::string result = aWriter.to_string();
    json_writer::insert_emacs_indent_hint(result, aWriter.indent());
    return result;
}

// ----------------------------------------------------------------------

namespace json_writer
{
    template <typename V> inline std::string pretty_json(const V& value, std::string keyword, size_t indent)
    {
        pretty aWriter{keyword, indent};
        aWriter << value;
        return aWriter << finalize;
    }

    template <typename V> inline std::string compact_json(const V& aValue, std::string keyword)
    {
        compact aWriter{keyword};
        aWriter << aValue;
        return aWriter << finalize;
    }

    template <typename V> inline std::string json(const V& value, std::string keyword, size_t indent)
    {
        if (indent)
            return pretty_json(value, keyword, indent);
        else
            return compact_json(value, keyword);
    }

      // ----------------------------------------------------------------------

#ifdef ACMACSD_FILESYSTEM
    template <typename V> inline void export_to_json(const V& value, std::string keyword, std::string filename, size_t indent, bool insert_emacs_indent_hint = true, bool force_compression = false)
    {
        acmacs_base::write_file(filename, json(value, keyword, indent, insert_emacs_indent_hint), force_compression);
    }
#endif

} // namespace json_writer

// ----------------------------------------------------------------------
/// Local Variables:
/// eval: (if (fboundp 'eu-rename-buffer) (eu-rename-buffer))
/// End:
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2019 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <boost/type.hpp>
#include <random>
#include <unordered_set>
#include <seastar/core/sleep.hh>

#include "keys.hh"
#include "schema_builder.hh"
#include "db/config.hh"
#include "db/system_keyspace.hh"
#include "db/system_distributed_keyspace.hh"
#include "dht/token-sharding.hh"
#include "locator/token_metadata.hh"
#include "gms/application_state.hh"
#include "gms/inet_address.hh"
#include "gms/gossiper.hh"

#include "cdc/generation.hh"

extern logging::logger cdc_log;

static int get_shard_count(const gms::inet_address& endpoint, const gms::gossiper& g) {
    auto ep_state = g.get_application_state_ptr(endpoint, gms::application_state::SHARD_COUNT);
    return ep_state ? std::stoi(ep_state->value) : -1;
}

static unsigned get_sharding_ignore_msb(const gms::inet_address& endpoint, const gms::gossiper& g) {
    auto ep_state = g.get_application_state_ptr(endpoint, gms::application_state::IGNORE_MSB_BITS);
    return ep_state ? std::stoi(ep_state->value) : 0;
}

namespace cdc {

extern const api::timestamp_clock::duration generation_leeway =
    std::chrono::duration_cast<api::timestamp_clock::duration>(std::chrono::seconds(5));

static void copy_int_to_bytes(int64_t i, size_t offset, bytes& b) {
    i = net::hton(i);
    std::copy_n(reinterpret_cast<int8_t*>(&i), sizeof(int64_t), b.begin() + offset);
}

stream_id::stream_id(int64_t first, int64_t second)
    : _value(bytes::initialized_later(), 2 * sizeof(int64_t))
{
    copy_int_to_bytes(first, 0, _value);
    copy_int_to_bytes(second, sizeof(int64_t), _value);
}

stream_id::stream_id(bytes b) : _value(std::move(b)) { }

bool stream_id::is_set() const {
    return !_value.empty();
}

bool stream_id::operator==(const stream_id& o) const {
    return _value == o._value;
}

bool stream_id::operator!=(const stream_id& o) const {
    return !(*this == o);
}

bool stream_id::operator<(const stream_id& o) const {
    return _value < o._value;
}

static int64_t bytes_to_int64(bytes_view b, size_t offset) {
    assert(b.size() >= offset + sizeof(int64_t));
    int64_t res;
    std::copy_n(b.begin() + offset, sizeof(int64_t), reinterpret_cast<int8_t *>(&res));
    return net::ntoh(res);
}

int64_t stream_id::first() const {
    return token_from_bytes(_value);
}

int64_t stream_id::second() const {
    return bytes_to_int64(_value, sizeof(int64_t));
}

int64_t stream_id::token_from_bytes(bytes_view b) {
    return bytes_to_int64(b, 0);
}

const bytes& stream_id::to_bytes() const {
    return _value;
}

partition_key stream_id::to_partition_key(const schema& log_schema) const {
    return partition_key::from_single_value(log_schema, _value);
}

bool token_range_description::operator==(const token_range_description& o) const {
    return token_range_end == o.token_range_end && streams == o.streams
        && sharding_ignore_msb == o.sharding_ignore_msb;
}

topology_description::topology_description(std::vector<token_range_description> entries)
    : _entries(std::move(entries)) {}

bool topology_description::operator==(const topology_description& o) const {
    return _entries == o._entries;
}

const std::vector<token_range_description>& topology_description::entries() const {
    return _entries;
}

static stream_id create_stream_id(dht::token t) {
    static thread_local std::mt19937_64 rand_gen(std::random_device().operator()());
    static thread_local std::uniform_int_distribution<int64_t> rand_dist(std::numeric_limits<int64_t>::min());

    return {dht::token::to_int64(t), rand_dist(rand_gen)};
}

class topology_description_generator final {
    const db::config& _cfg;
    const std::unordered_set<dht::token>& _bootstrap_tokens;
    const locator::token_metadata& _token_metadata;
    const gms::gossiper& _gossiper;

    // Compute a set of tokens that split the token ring into vnodes
    auto get_tokens() const {
        auto tokens = _token_metadata.sorted_tokens();
        auto it = tokens.insert(
                tokens.end(), _bootstrap_tokens.begin(), _bootstrap_tokens.end());
        std::sort(it, tokens.end());
        std::inplace_merge(tokens.begin(), it, tokens.end());
        tokens.erase(std::unique(tokens.begin(), tokens.end()), tokens.end());
        return tokens;
    }

    // Fetch sharding parameters for a node that owns vnode ending with this.end
    // Returns <shard_count, ignore_msb> pair.
    std::pair<size_t, uint8_t> get_sharding_info(dht::token end) const {
        if (_bootstrap_tokens.count(end) > 0) {
            return {smp::count, _cfg.murmur3_partitioner_ignore_msb_bits()};
        } else {
            auto endpoint = _token_metadata.get_endpoint(end);
            if (!endpoint) {
                throw std::runtime_error(
                        format("Can't find endpoint for token {}", end));
            }
            auto sc = get_shard_count(*endpoint, _gossiper);
            return {sc > 0 ? sc : 1, get_sharding_ignore_msb(*endpoint, _gossiper)};
        }
    }

    token_range_description create_description(dht::token start, dht::token end) const {
        token_range_description desc;

        desc.token_range_end = end;

        auto [shard_count, ignore_msb] = get_sharding_info(end);
        desc.streams.reserve(shard_count);
        desc.sharding_ignore_msb = ignore_msb;

        dht::sharder sharder(shard_count, ignore_msb);
        for (size_t shard_idx = 0; shard_idx < shard_count; ++shard_idx) {
            auto t = dht::find_first_token_for_shard(sharder, start, end, shard_idx);
            desc.streams.push_back(create_stream_id(t));
        }

        return desc;
    }
public:
    topology_description_generator(
            const db::config& cfg,
            const std::unordered_set<dht::token>& bootstrap_tokens,
            const locator::token_metadata& token_metadata,
            const gms::gossiper& gossiper)
        : _cfg(cfg)
        , _bootstrap_tokens(bootstrap_tokens)
        , _token_metadata(token_metadata)
        , _gossiper(gossiper)
    {}

    /*
     * Generate a set of CDC stream identifiers such that for each shard
     * and vnode pair there exists a stream whose token falls into this vnode
     * and is owned by this shard. It is sometimes not possible to generate
     * a CDC stream identifier for some (vnode, shard) pair because not all
     * shards have to own tokens in a vnode. Small vnode can be totally owned
     * by a single shard. In such case, a stream identifier that maps to
     * end of the vnode is generated.
     *
     * Then build a cdc::topology_description which maps tokens to generated
     * stream identifiers, such that if token T is owned by shard S in vnode V,
     * it gets mapped to the stream identifier generated for (S, V).
     */
    // Run in seastar::async context.
    topology_description generate() const {
        const auto tokens = get_tokens();

        std::vector<token_range_description> vnode_descriptions;
        vnode_descriptions.reserve(tokens.size());

        vnode_descriptions.push_back(
                create_description(tokens.back(), tokens.front()));
        for (size_t idx = 1; idx < tokens.size(); ++idx) {
            vnode_descriptions.push_back(
                    create_description(tokens[idx - 1], tokens[idx]));
        }

        return {std::move(vnode_descriptions)};
    }
};

bool should_propose_first_generation(const gms::inet_address& me, const gms::gossiper& g) {
    auto my_host_id = g.get_host_id(me);
    auto& eps = g.get_endpoint_states();
    return std::none_of(eps.begin(), eps.end(),
            [&] (const std::pair<gms::inet_address, gms::endpoint_state>& ep) {
        return my_host_id < g.get_host_id(ep.first);
    });
}

future<db_clock::time_point> get_local_streams_timestamp() {
    return db::system_keyspace::get_saved_cdc_streams_timestamp().then([] (std::optional<db_clock::time_point> ts) {
        if (!ts) {
            auto err = format("get_local_streams_timestamp: tried to retrieve streams timestamp after bootstrapping, but it's not present");
            cdc_log.error("{}", err);
            throw std::runtime_error(err);
        }
        return *ts;
    });
}

// Run inside seastar::async context.
db_clock::time_point make_new_cdc_generation(
        const db::config& cfg,
        const std::unordered_set<dht::token>& bootstrap_tokens,
        const locator::token_metadata& tm,
        const gms::gossiper& g,
        db::system_distributed_keyspace& sys_dist_ks,
        std::chrono::milliseconds ring_delay,
        bool for_testing) {
    auto gen = topology_description_generator(cfg, bootstrap_tokens, tm, g).generate();

    // Begin the race.
    auto ts = db_clock::now() + (
            for_testing ? std::chrono::milliseconds(0) : (
                2 * ring_delay + std::chrono::duration_cast<std::chrono::milliseconds>(generation_leeway)));
    sys_dist_ks.insert_cdc_topology_description(ts, std::move(gen), { tm.count_normal_token_owners() }).get();

    return ts;
}

std::optional<db_clock::time_point> get_streams_timestamp_for(const gms::inet_address& endpoint, const gms::gossiper& g) {
    auto streams_ts_string = g.get_application_state_value(endpoint, gms::application_state::CDC_STREAMS_TIMESTAMP);
    cdc_log.trace("endpoint={}, streams_ts_string={}", endpoint, streams_ts_string);
    return gms::versioned_value::cdc_streams_timestamp_from_string(streams_ts_string);
}

// Run inside seastar::async context.
static void do_update_streams_description(
        db_clock::time_point streams_ts,
        db::system_distributed_keyspace& sys_dist_ks,
        db::system_distributed_keyspace::context ctx) {
    if (sys_dist_ks.cdc_desc_exists(streams_ts, ctx).get0()) {
        cdc_log.debug("update_streams_description: description of generation {} already inserted", streams_ts);
        return;
    }

    // We might race with another node also inserting the description, but that's ok. It's an idempotent operation.

    auto topo = sys_dist_ks.read_cdc_topology_description(streams_ts, ctx).get0();
    if (!topo) {
        throw std::runtime_error(format("could not find streams data for timestamp {}", streams_ts));
    }

    std::set<cdc::stream_id> streams_set;
    for (auto& entry: topo->entries()) {
        streams_set.insert(entry.streams.begin(), entry.streams.end());
    }

    std::vector<cdc::stream_id> streams_vec(streams_set.begin(), streams_set.end());

    sys_dist_ks.create_cdc_desc(streams_ts, streams_vec, ctx).get();
    cdc_log.info("CDC description table successfully updated with generation {}.", streams_ts);
}

void update_streams_description(
        db_clock::time_point streams_ts,
        shared_ptr<db::system_distributed_keyspace> sys_dist_ks,
        noncopyable_function<unsigned()> get_num_token_owners,
        abort_source& abort_src) {
    try {
        do_update_streams_description(streams_ts, *sys_dist_ks, { get_num_token_owners() });
    } catch(...) {
        cdc_log.warn(
            "Could not update CDC description table with generation {}: {}. Will retry in the background.",
            streams_ts, std::current_exception());

        // It is safe to discard this future: we keep system distributed keyspace alive.
        (void)seastar::async([
            streams_ts, sys_dist_ks, get_num_token_owners = std::move(get_num_token_owners), &abort_src
        ] {
            while (true) {
                sleep_abortable(std::chrono::seconds(60), abort_src).get();
                try {
                    do_update_streams_description(streams_ts, *sys_dist_ks, { get_num_token_owners() });
                    return;
                } catch (...) {
                    cdc_log.warn(
                        "Could not update CDC description table with generation {}: {}. Will try again.",
                        streams_ts, std::current_exception());
                }
            }
        });
    }
}

} // namespace cdc
<commit_msg>cdc: if ring_delay == 0, don't add delay to newly created generation<commit_after>/*
 * Copyright (C) 2019 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <boost/type.hpp>
#include <random>
#include <unordered_set>
#include <seastar/core/sleep.hh>

#include "keys.hh"
#include "schema_builder.hh"
#include "db/config.hh"
#include "db/system_keyspace.hh"
#include "db/system_distributed_keyspace.hh"
#include "dht/token-sharding.hh"
#include "locator/token_metadata.hh"
#include "gms/application_state.hh"
#include "gms/inet_address.hh"
#include "gms/gossiper.hh"

#include "cdc/generation.hh"

extern logging::logger cdc_log;

static int get_shard_count(const gms::inet_address& endpoint, const gms::gossiper& g) {
    auto ep_state = g.get_application_state_ptr(endpoint, gms::application_state::SHARD_COUNT);
    return ep_state ? std::stoi(ep_state->value) : -1;
}

static unsigned get_sharding_ignore_msb(const gms::inet_address& endpoint, const gms::gossiper& g) {
    auto ep_state = g.get_application_state_ptr(endpoint, gms::application_state::IGNORE_MSB_BITS);
    return ep_state ? std::stoi(ep_state->value) : 0;
}

namespace cdc {

extern const api::timestamp_clock::duration generation_leeway =
    std::chrono::duration_cast<api::timestamp_clock::duration>(std::chrono::seconds(5));

static void copy_int_to_bytes(int64_t i, size_t offset, bytes& b) {
    i = net::hton(i);
    std::copy_n(reinterpret_cast<int8_t*>(&i), sizeof(int64_t), b.begin() + offset);
}

stream_id::stream_id(int64_t first, int64_t second)
    : _value(bytes::initialized_later(), 2 * sizeof(int64_t))
{
    copy_int_to_bytes(first, 0, _value);
    copy_int_to_bytes(second, sizeof(int64_t), _value);
}

stream_id::stream_id(bytes b) : _value(std::move(b)) { }

bool stream_id::is_set() const {
    return !_value.empty();
}

bool stream_id::operator==(const stream_id& o) const {
    return _value == o._value;
}

bool stream_id::operator!=(const stream_id& o) const {
    return !(*this == o);
}

bool stream_id::operator<(const stream_id& o) const {
    return _value < o._value;
}

static int64_t bytes_to_int64(bytes_view b, size_t offset) {
    assert(b.size() >= offset + sizeof(int64_t));
    int64_t res;
    std::copy_n(b.begin() + offset, sizeof(int64_t), reinterpret_cast<int8_t *>(&res));
    return net::ntoh(res);
}

int64_t stream_id::first() const {
    return token_from_bytes(_value);
}

int64_t stream_id::second() const {
    return bytes_to_int64(_value, sizeof(int64_t));
}

int64_t stream_id::token_from_bytes(bytes_view b) {
    return bytes_to_int64(b, 0);
}

const bytes& stream_id::to_bytes() const {
    return _value;
}

partition_key stream_id::to_partition_key(const schema& log_schema) const {
    return partition_key::from_single_value(log_schema, _value);
}

bool token_range_description::operator==(const token_range_description& o) const {
    return token_range_end == o.token_range_end && streams == o.streams
        && sharding_ignore_msb == o.sharding_ignore_msb;
}

topology_description::topology_description(std::vector<token_range_description> entries)
    : _entries(std::move(entries)) {}

bool topology_description::operator==(const topology_description& o) const {
    return _entries == o._entries;
}

const std::vector<token_range_description>& topology_description::entries() const {
    return _entries;
}

static stream_id create_stream_id(dht::token t) {
    static thread_local std::mt19937_64 rand_gen(std::random_device().operator()());
    static thread_local std::uniform_int_distribution<int64_t> rand_dist(std::numeric_limits<int64_t>::min());

    return {dht::token::to_int64(t), rand_dist(rand_gen)};
}

class topology_description_generator final {
    const db::config& _cfg;
    const std::unordered_set<dht::token>& _bootstrap_tokens;
    const locator::token_metadata& _token_metadata;
    const gms::gossiper& _gossiper;

    // Compute a set of tokens that split the token ring into vnodes
    auto get_tokens() const {
        auto tokens = _token_metadata.sorted_tokens();
        auto it = tokens.insert(
                tokens.end(), _bootstrap_tokens.begin(), _bootstrap_tokens.end());
        std::sort(it, tokens.end());
        std::inplace_merge(tokens.begin(), it, tokens.end());
        tokens.erase(std::unique(tokens.begin(), tokens.end()), tokens.end());
        return tokens;
    }

    // Fetch sharding parameters for a node that owns vnode ending with this.end
    // Returns <shard_count, ignore_msb> pair.
    std::pair<size_t, uint8_t> get_sharding_info(dht::token end) const {
        if (_bootstrap_tokens.count(end) > 0) {
            return {smp::count, _cfg.murmur3_partitioner_ignore_msb_bits()};
        } else {
            auto endpoint = _token_metadata.get_endpoint(end);
            if (!endpoint) {
                throw std::runtime_error(
                        format("Can't find endpoint for token {}", end));
            }
            auto sc = get_shard_count(*endpoint, _gossiper);
            return {sc > 0 ? sc : 1, get_sharding_ignore_msb(*endpoint, _gossiper)};
        }
    }

    token_range_description create_description(dht::token start, dht::token end) const {
        token_range_description desc;

        desc.token_range_end = end;

        auto [shard_count, ignore_msb] = get_sharding_info(end);
        desc.streams.reserve(shard_count);
        desc.sharding_ignore_msb = ignore_msb;

        dht::sharder sharder(shard_count, ignore_msb);
        for (size_t shard_idx = 0; shard_idx < shard_count; ++shard_idx) {
            auto t = dht::find_first_token_for_shard(sharder, start, end, shard_idx);
            desc.streams.push_back(create_stream_id(t));
        }

        return desc;
    }
public:
    topology_description_generator(
            const db::config& cfg,
            const std::unordered_set<dht::token>& bootstrap_tokens,
            const locator::token_metadata& token_metadata,
            const gms::gossiper& gossiper)
        : _cfg(cfg)
        , _bootstrap_tokens(bootstrap_tokens)
        , _token_metadata(token_metadata)
        , _gossiper(gossiper)
    {}

    /*
     * Generate a set of CDC stream identifiers such that for each shard
     * and vnode pair there exists a stream whose token falls into this vnode
     * and is owned by this shard. It is sometimes not possible to generate
     * a CDC stream identifier for some (vnode, shard) pair because not all
     * shards have to own tokens in a vnode. Small vnode can be totally owned
     * by a single shard. In such case, a stream identifier that maps to
     * end of the vnode is generated.
     *
     * Then build a cdc::topology_description which maps tokens to generated
     * stream identifiers, such that if token T is owned by shard S in vnode V,
     * it gets mapped to the stream identifier generated for (S, V).
     */
    // Run in seastar::async context.
    topology_description generate() const {
        const auto tokens = get_tokens();

        std::vector<token_range_description> vnode_descriptions;
        vnode_descriptions.reserve(tokens.size());

        vnode_descriptions.push_back(
                create_description(tokens.back(), tokens.front()));
        for (size_t idx = 1; idx < tokens.size(); ++idx) {
            vnode_descriptions.push_back(
                    create_description(tokens[idx - 1], tokens[idx]));
        }

        return {std::move(vnode_descriptions)};
    }
};

bool should_propose_first_generation(const gms::inet_address& me, const gms::gossiper& g) {
    auto my_host_id = g.get_host_id(me);
    auto& eps = g.get_endpoint_states();
    return std::none_of(eps.begin(), eps.end(),
            [&] (const std::pair<gms::inet_address, gms::endpoint_state>& ep) {
        return my_host_id < g.get_host_id(ep.first);
    });
}

future<db_clock::time_point> get_local_streams_timestamp() {
    return db::system_keyspace::get_saved_cdc_streams_timestamp().then([] (std::optional<db_clock::time_point> ts) {
        if (!ts) {
            auto err = format("get_local_streams_timestamp: tried to retrieve streams timestamp after bootstrapping, but it's not present");
            cdc_log.error("{}", err);
            throw std::runtime_error(err);
        }
        return *ts;
    });
}

// Run inside seastar::async context.
db_clock::time_point make_new_cdc_generation(
        const db::config& cfg,
        const std::unordered_set<dht::token>& bootstrap_tokens,
        const locator::token_metadata& tm,
        const gms::gossiper& g,
        db::system_distributed_keyspace& sys_dist_ks,
        std::chrono::milliseconds ring_delay,
        bool for_testing) {
    using namespace std::chrono;
    auto gen = topology_description_generator(cfg, bootstrap_tokens, tm, g).generate();

    // Begin the race.
    auto ts = db_clock::now() + (
            (for_testing || ring_delay == milliseconds(0)) ? milliseconds(0) : (
                2 * ring_delay + duration_cast<milliseconds>(generation_leeway)));
    sys_dist_ks.insert_cdc_topology_description(ts, std::move(gen), { tm.count_normal_token_owners() }).get();

    return ts;
}

std::optional<db_clock::time_point> get_streams_timestamp_for(const gms::inet_address& endpoint, const gms::gossiper& g) {
    auto streams_ts_string = g.get_application_state_value(endpoint, gms::application_state::CDC_STREAMS_TIMESTAMP);
    cdc_log.trace("endpoint={}, streams_ts_string={}", endpoint, streams_ts_string);
    return gms::versioned_value::cdc_streams_timestamp_from_string(streams_ts_string);
}

// Run inside seastar::async context.
static void do_update_streams_description(
        db_clock::time_point streams_ts,
        db::system_distributed_keyspace& sys_dist_ks,
        db::system_distributed_keyspace::context ctx) {
    if (sys_dist_ks.cdc_desc_exists(streams_ts, ctx).get0()) {
        cdc_log.debug("update_streams_description: description of generation {} already inserted", streams_ts);
        return;
    }

    // We might race with another node also inserting the description, but that's ok. It's an idempotent operation.

    auto topo = sys_dist_ks.read_cdc_topology_description(streams_ts, ctx).get0();
    if (!topo) {
        throw std::runtime_error(format("could not find streams data for timestamp {}", streams_ts));
    }

    std::set<cdc::stream_id> streams_set;
    for (auto& entry: topo->entries()) {
        streams_set.insert(entry.streams.begin(), entry.streams.end());
    }

    std::vector<cdc::stream_id> streams_vec(streams_set.begin(), streams_set.end());

    sys_dist_ks.create_cdc_desc(streams_ts, streams_vec, ctx).get();
    cdc_log.info("CDC description table successfully updated with generation {}.", streams_ts);
}

void update_streams_description(
        db_clock::time_point streams_ts,
        shared_ptr<db::system_distributed_keyspace> sys_dist_ks,
        noncopyable_function<unsigned()> get_num_token_owners,
        abort_source& abort_src) {
    try {
        do_update_streams_description(streams_ts, *sys_dist_ks, { get_num_token_owners() });
    } catch(...) {
        cdc_log.warn(
            "Could not update CDC description table with generation {}: {}. Will retry in the background.",
            streams_ts, std::current_exception());

        // It is safe to discard this future: we keep system distributed keyspace alive.
        (void)seastar::async([
            streams_ts, sys_dist_ks, get_num_token_owners = std::move(get_num_token_owners), &abort_src
        ] {
            while (true) {
                sleep_abortable(std::chrono::seconds(60), abort_src).get();
                try {
                    do_update_streams_description(streams_ts, *sys_dist_ks, { get_num_token_owners() });
                    return;
                } catch (...) {
                    cdc_log.warn(
                        "Could not update CDC description table with generation {}: {}. Will try again.",
                        streams_ts, std::current_exception());
                }
            }
        });
    }
}

} // namespace cdc
<|endoftext|>