<commit_before>#include "game.hpp"

int64_t Game::negamax(BitBoard & b, int64_t alpha, int64_t beta, int depth, int real_depth, int rule, bool inNullMove, bool quies_find, PV* pline) {
	++nodesCounter;
	/*if(game_board.testOfDraw()) {
		return 0;
	}*/

	if(depth >= 5) {
		if(is_input_available()) {
			std::string input_str;
			std::getline(std::cin, input_str);
			if(input_str == "stop") {
				stopped = true;
			} else if(input_str == "isready") {
				std::cout << "readyok\n";
			}
		}
	}

	int64_t oldAlpha = alpha;

	bool extended = false;

	int64_t eval = -WHITE_WIN;
	double margin = PAWN_EV / 2;

	int nextDepth = depth - 1;
	if(depth > 2) {
		if(rule == FIXED_TIME && timer.getTime() >= time || stopped) {
			return 0;
		}
	}

	uint8_t color;
	if(b.whiteMove) {
		color = WHITE;
	} else {
		color = BLACK;
	}

	if(depth <= 0 || real_depth >= 100) {
		pline->count = 0;

		if(quies_find) {
			return b.getEvalute();
		}
		return quies(b, alpha, beta, rule, real_depth);
	}

	//PV line;

	if(b.inCheck(color)) {
		++nextDepth;
		extended = true;
		inNullMove = true;
	}

	int tmp;

	uint64_t hash = b.getColorHash();
	Hash* currentHash = &boardHash[hash & hash_cutter];

	if(currentHash->flag != EMPTY && currentHash->key == hash && !extended) {
		if(real_depth > 0 && currentHash->depth >= depth) {
			double score = currentHash->score;

			if(score > WHITE_WIN - 100) {
				score -= real_depth;
			} else if(score < -WHITE_WIN + 100) {
				score += real_depth;
			}

			if(currentHash->flag == EXACT || currentHash->flag == BETA) {
				if(score > alpha) {
					alpha = score;
				}
				if(alpha >= beta) {
					return beta;
				}
			} else if(currentHash->flag == ALPHA) {
				if(score <= alpha) {
					return alpha;
				}
			}
		}

		if(currentHash->flag != ALPHA && real_depth > 0 && !quies_find) {
			b.move(currentHash->move);
			tmp = -negamax(b, -beta, -alpha, depth - 1, real_depth + 1, rule, inNullMove, false, pline);
			b.goBack();

			if(tmp > alpha) {
				recordHash(depth, tmp, tmp<beta?EXACT:BETA, hash, currentHash->move, real_depth);
				alpha = tmp;
				if(alpha >= beta) {
		  			return beta;
				}
			}
		}
	}

	bool inCheck;
	if(color == WHITE) {
		inCheck = b.inCheck(WHITE);
	} else {
		inCheck = b.inCheck(BLACK);
	}

	if(option.nullMovePrunningEnable) {
		if(!quies_find && !inNullMove && !b.inCheck(color) && !extended && !b.attacked && real_depth > 4 /*&& b.getFiguresCount() > 3*/) {
			b.whiteMove = !b.whiteMove;
			int R;
			if(depth > 6) {
				R = 4;
			} else {
				R = 3;
			}

			double value = -negamax(b, -beta, -(beta - 1), depth - 1 - R, real_depth + 1, rule, true, quies_find, pline);
			if(value >= beta) {
				b.whiteMove = !b.whiteMove;
				return value;
			}
			b.whiteMove = !b.whiteMove;
		}
	}

	int num_moves = 0;

	b.bitBoardMoveGenerator(moveArray[real_depth]);
	sortAttacks(moveArray[real_depth]);
	sortMoves(moveArray[real_depth], real_depth);

	BitMove local_move;

	if(moveArray[real_depth].count > 0) {
		local_move = moveArray[real_depth].moveArray[0];
	}

	for(unsigned int i = 0; i < moveArray[real_depth].count; ++i) {
		b.move(moveArray[real_depth].moveArray[i]);

		if(game_board.inCheck(color)) {
			b.goBack();
			continue;
		}

		if(num_moves == 0) {
			local_move = moveArray[real_depth].moveArray[i];
		}

		++num_moves;

		if(real_depth == 0 && depth >= 9) {
			std::cout << "info currmove " << moveArray[real_depth].moveArray[i].getMoveString() << " currmovenumber " << num_moves << std::endl;
		}

		if(!option.lmrEnable) {
			tmp = -negamax(b, -(alpha + 1), -alpha, nextDepth, real_depth + 1, rule, inNullMove, quies_find, pline);

			if(tmp > alpha && tmp < beta) {
				tmp = -negamax(b, -beta, -alpha, nextDepth, real_depth + 1, rule, inNullMove, quies_find, pline);
			}

		} else {
			if(num_moves <= 3 || b.inCheck(color) || extended || inNullMove || b.attacked) {
				tmp = -negamax(b, -beta, -alpha, nextDepth, real_depth + 1, rule, inNullMove, quies_find, pline);
			} else {
				tmp = -negamax(b, -(alpha + 1), -alpha, nextDepth-1, real_depth + 1, rule, inNullMove, quies_find, pline);

				if(tmp > alpha) {
					tmp = -negamax(b, -beta, -alpha, nextDepth, real_depth + 1, rule, inNullMove, quies_find, pline);
				}
			}
		}

		b.goBack();

		if(tmp > eval) {
			eval = tmp;
		}

		if(tmp > alpha) {
			alpha = tmp;
			local_move = moveArray[real_depth].moveArray[i];

			if(!local_move.isAttack) {
				if(color == WHITE) {
					//whiteHistorySort[local_move.fromY][local_move.fromX][local_move.toY][local_move.toX] += pow(depth, 2);
					whiteKiller[real_depth] = Killer(local_move);
				} else {
					//blackHistorySort[local_move.fromY][local_move.fromX][local_move.toY][local_move.toX] += pow(depth, 2);
					blackKiller[real_depth] = Killer(local_move);
				}
			}

			recordHash(depth, tmp, tmp<beta?EXACT:BETA, hash, moveArray[real_depth].moveArray[i], real_depth);

			//pline->line[0] = local_move;
			//memcpy(pline->line + 1, line.line, line.count * sizeof(local_move));
			//pline->count = line.count + 1;
		}

		if(alpha >= beta) {
			return beta;
		}
	}

	if(num_moves == 0) {
		if(game_board.inCheck(color)) {
			return BLACK_WIN + real_depth;
		} else {
			return 0;
		}
	}

	if(real_depth == 0) {
		if(rule == FIXED_TIME && timer.getTime() >= time || stopped) {
			return 0;
		}

		bestMove = local_move;
		bestScore = alpha;

		//for(int i = 0; i < pline->count; ++i) {
			//std::cout << pline->line[i].getMoveString() << " ";
		//}
	}

	if(alpha == oldAlpha) {
		recordHash(depth, alpha, ALPHA, hash, local_move, real_depth);
	}

	if(real_depth == 0) {
		if(num_moves >= 0) {
			std::cout << "info depth " << max_depth << " time " << (int)((clock() - start_timer) / (CLOCKS_PER_SEC / 1000)) << " nodes " << nodesCounter;
			std::cout << " ";
			printScore(eval);
			std::cout << " pv " << local_move.getMoveString() << " nps " << (int)(nodesCounter / ((clock() - start_timer) / CLOCKS_PER_SEC)) << std::endl;
		} else {
			std::cout << std::endl;
		}
	}

	return alpha;
}

uint64_t Game::perft(int depth) {
	uint64_t res = 0;
	if(depth == 0) {
		return 1;
	}

	game_board.bitBoardMoveGenerator(moveArray[depth]);

	for(unsigned int i = 0; i < moveArray[depth].count; ++i) {
		game_board.move(moveArray[depth].moveArray[i]);
		if(game_board.whiteMove) {
			if(game_board.inCheck(BLACK)) {
				game_board.goBack();
				continue;
			}
		} else {
			if(game_board.inCheck(WHITE)) {
				game_board.goBack();
				continue;
			}
		}

		res += perft(depth - 1);
		game_board.goBack();
	}

	return res;
}

int64_t Game::quies(BitBoard & b, int64_t alpha, int64_t beta, int rule, int real_depth) {
	int64_t val = b.getEvalute();

	uint8_t color;

	if(b.whiteMove) {
		color = WHITE;
	} else {
		color = BLACK;
	}

	/*if(b.inCheck(color)) {
		return negamax(b, alpha, beta, 1, real_depth, rule, false, true, new PV());
	}*/

	if(val >= beta) {
		return val;
	}

	if(alpha < val) {
		alpha = val;
	}

	b.bitBoardAttackMoveGenerator(moveArray[real_depth]);
	sortAttacks(moveArray[real_depth]);

	for(unsigned int i = 0; i < moveArray[real_depth].count; ++i) {
		++nodesCounter;
		b.move(moveArray[real_depth].moveArray[i]);

		if(!b.whiteMove) {
			if(game_board.inCheck(WHITE)) {
				b.goBack();
				continue;
			}
		} else {
			if(game_board.inCheck(BLACK)) {
				b.goBack();
				continue;
			}
		}

		val = -quies(b, -beta, -alpha, rule, real_depth + 1);
		b.goBack();

		if(val > alpha) {
			alpha = val;
		}

		if(val >= beta) {
			return beta;
		}
	}

	return alpha;
}

bool Game::recordHash(int depth, int score, int flag, uint64_t key, BitMove move, int real_depth) {
	Hash* hash = &boardHash[key & hash_cutter];

	if(flag == ALPHA && (hash->flag == EXACT || hash->flag == BETA)) {
		return 0;
	}

	if(hash->flag != EMPTY && hash->depth > depth) {
		return 0;
	}

	if(score > WHITE_WIN - 100) {
		score += real_depth;
	} else if(score < -WHITE_WIN + 100) {
		score -= real_depth;
	}

	hash->depth = depth;
	hash->score = score;
	hash->flag = flag;
	hash->key = key;

	if(flag != ALPHA) {
		hash->move = move;
	}
}
<commit_msg>fixed bugs<commit_after>#include "game.hpp"

int64_t Game::negamax(BitBoard & b, int64_t alpha, int64_t beta, int depth, int real_depth, int rule, bool inNullMove, bool quies_find, PV* pline) {
	++nodesCounter;
	/*if(game_board.testOfDraw()) {
		return 0;
	}*/

	if(depth >= 5) {
		if(is_input_available()) {
			std::string input_str;
			std::getline(std::cin, input_str);
			if(input_str == "stop") {
				stopped = true;
			} else if(input_str == "isready") {
				std::cout << "readyok\n";
			}
		}
	}

	int64_t oldAlpha = alpha;

	bool extended = false;

	int64_t eval = -WHITE_WIN;
	double margin = PAWN_EV / 2;

	int nextDepth = depth - 1;
	if(depth > 2) {
		if(rule == FIXED_TIME && timer.getTime() >= time || stopped) {
			return 0;
		}
	}

	uint8_t color;
	if(b.whiteMove) {
		color = WHITE;
	} else {
		color = BLACK;
	}

	if(depth <= 0 || real_depth >= 100) {
		pline->count = 0;

		if(quies_find) {
			return b.getEvalute();
		}
		return quies(b, alpha, beta, rule, real_depth);
	}

	//PV line;

	if(b.inCheck(color)) {
		++nextDepth;
		extended = true;
		inNullMove = true;
	}

	int tmp;

	uint64_t hash = b.getColorHash();
	Hash* currentHash = &boardHash[hash & hash_cutter];

	if(currentHash->flag != EMPTY && currentHash->key == hash && !extended) {
		if(real_depth > 0 && currentHash->depth >= depth) {
			double score = currentHash->score;

			if(score > WHITE_WIN - 100) {
				score -= real_depth;
			} else if(score < -WHITE_WIN + 100) {
				score += real_depth;
			}

			if(currentHash->flag == EXACT || currentHash->flag == BETA) {
				if(score > alpha) {
					alpha = score;
				}
				if(alpha >= beta) {
					return beta;
				}
			} else if(currentHash->flag == ALPHA) {
				if(score <= alpha) {
					return alpha;
				}
			}
		}

		if(currentHash->flag != ALPHA && real_depth > 0 && !quies_find) {
			b.move(currentHash->move);
			tmp = -negamax(b, -beta, -alpha, depth - 1, real_depth + 1, rule, inNullMove, false, pline);
			b.goBack();

			if(tmp > alpha) {
				recordHash(depth, tmp, tmp<beta?EXACT:BETA, hash, currentHash->move, real_depth);
				alpha = tmp;
				if(alpha >= beta) {
		  			return beta;
				}
			}
		}
	}

	bool inCheck;
	if(color == WHITE) {
		inCheck = b.inCheck(WHITE);
	} else {
		inCheck = b.inCheck(BLACK);
	}

	if(option.nullMovePrunningEnable) {
		if(!quies_find && !inNullMove && !b.inCheck(color) && !extended && !b.attacked && real_depth > 4 /*&& b.getFiguresCount() > 3*/) {
			b.whiteMove = !b.whiteMove;
			int R;
			if(depth > 6) {
				R = 4;
			} else {
				R = 3;
			}

			double value = -negamax(b, -beta, -(beta - 1), depth - 1 - R, real_depth + 1, rule, true, quies_find, pline);
			if(value >= beta) {
				b.whiteMove = !b.whiteMove;
				return value;
			}
			b.whiteMove = !b.whiteMove;
		}
	}

	int num_moves = 0;

	b.bitBoardMoveGenerator(moveArray[real_depth]);
	sortAttacks(moveArray[real_depth]);
	sortMoves(moveArray[real_depth], real_depth);

	BitMove local_move;

	if(moveArray[real_depth].count > 0) {
		local_move = moveArray[real_depth].moveArray[0];
	}

	for(unsigned int i = 0; i < moveArray[real_depth].count; ++i) {
		b.move(moveArray[real_depth].moveArray[i]);

		if(game_board.inCheck(color)) {
			b.goBack();
			continue;
		}

		if(num_moves == 0) {
			local_move = moveArray[real_depth].moveArray[i];
		}

		++num_moves;

		if(real_depth == 0 && depth >= 9) {
			std::cout << "info currmove " << moveArray[real_depth].moveArray[i].getMoveString() << " currmovenumber " << num_moves << std::endl;
		}

		if(!option.lmrEnable) {
			tmp = -negamax(b, -(alpha + 1), -alpha, nextDepth, real_depth + 1, rule, inNullMove, quies_find, pline);

			if(tmp > alpha && tmp < beta) {
				tmp = -negamax(b, -beta, -alpha, nextDepth, real_depth + 1, rule, inNullMove, quies_find, pline);
			}

		} else {
			if(num_moves <= 3 || b.inCheck(color) || extended || inNullMove || b.attacked) {
				tmp = -negamax(b, -beta, -alpha, nextDepth, real_depth + 1, rule, inNullMove, quies_find, pline);
			} else {
				tmp = -negamax(b, -(alpha + 1), -alpha, nextDepth-1, real_depth + 1, rule, inNullMove, quies_find, pline);

				if(tmp > alpha) {
					tmp = -negamax(b, -beta, -alpha, nextDepth, real_depth + 1, rule, inNullMove, quies_find, pline);
				}
			}
		}

		b.goBack();

		if(tmp > eval) {
			eval = tmp;
		}

		if(tmp > alpha) {
			alpha = tmp;
			local_move = moveArray[real_depth].moveArray[i];

			if(!local_move.isAttack) {
				if(color == WHITE) {
					//whiteHistorySort[local_move.fromY][local_move.fromX][local_move.toY][local_move.toX] += pow(depth, 2);
					whiteKiller[real_depth] = Killer(local_move);
				} else {
					//blackHistorySort[local_move.fromY][local_move.fromX][local_move.toY][local_move.toX] += pow(depth, 2);
					blackKiller[real_depth] = Killer(local_move);
				}
			}

			recordHash(depth, tmp, tmp<beta?EXACT:BETA, hash, moveArray[real_depth].moveArray[i], real_depth);

			//pline->line[0] = local_move;
			//memcpy(pline->line + 1, line.line, line.count * sizeof(local_move));
			//pline->count = line.count + 1;
		}

		if(alpha >= beta) {
			return beta;
		}
	}

	if(num_moves == 0) {
		if(game_board.inCheck(color)) {
			return BLACK_WIN + real_depth;
		} else {
			return 0;
		}
	}

	if(real_depth == 0) {
		if(rule == FIXED_TIME && timer.getTime() >= time || stopped) {
			return 0;
		}

		bestMove = local_move;
		bestScore = alpha;

		//for(int i = 0; i < pline->count; ++i) {
			//std::cout << pline->line[i].getMoveString() << " ";
		//}
	}

	if(alpha == oldAlpha) {
		recordHash(depth, alpha, ALPHA, hash, local_move, real_depth);
	}

	if(real_depth == 0) {
		if(num_moves >= 0) {
			std::cout << "info depth " << max_depth << " time " << (int)((clock() - start_timer) / (CLOCKS_PER_SEC / 1000)) << " nodes " << nodesCounter << " nps " << (int)(nodesCounter / ((clock() - start_timer) / CLOCKS_PER_SEC));
			std::cout << " ";
			printScore(eval);
			std::cout << " pv " << local_move.getMoveString() << std::endl;
		} else {
			std::cout << std::endl;
		}
	}

	return alpha;
}

uint64_t Game::perft(int depth) {
	uint64_t res = 0;
	if(depth == 0) {
		return 1;
	}

	game_board.bitBoardMoveGenerator(moveArray[depth]);

	for(unsigned int i = 0; i < moveArray[depth].count; ++i) {
		game_board.move(moveArray[depth].moveArray[i]);
		if(game_board.whiteMove) {
			if(game_board.inCheck(BLACK)) {
				game_board.goBack();
				continue;
			}
		} else {
			if(game_board.inCheck(WHITE)) {
				game_board.goBack();
				continue;
			}
		}

		res += perft(depth - 1);
		game_board.goBack();
	}

	return res;
}

int64_t Game::quies(BitBoard & b, int64_t alpha, int64_t beta, int rule, int real_depth) {
	int64_t val = b.getEvalute();

	uint8_t color;

	if(b.whiteMove) {
		color = WHITE;
	} else {
		color = BLACK;
	}

	/*if(b.inCheck(color)) {
		return negamax(b, alpha, beta, 1, real_depth, rule, false, true, new PV());
	}*/

	if(val >= beta) {
		return val;
	}

	if(alpha < val) {
		alpha = val;
	}

	b.bitBoardAttackMoveGenerator(moveArray[real_depth]);
	sortAttacks(moveArray[real_depth]);

	for(unsigned int i = 0; i < moveArray[real_depth].count; ++i) {
		++nodesCounter;
		b.move(moveArray[real_depth].moveArray[i]);

		if(!b.whiteMove) {
			if(game_board.inCheck(WHITE)) {
				b.goBack();
				continue;
			}
		} else {
			if(game_board.inCheck(BLACK)) {
				b.goBack();
				continue;
			}
		}

		val = -quies(b, -beta, -alpha, rule, real_depth + 1);
		b.goBack();

		if(val > alpha) {
			alpha = val;
		}

		if(val >= beta) {
			return beta;
		}
	}

	return alpha;
}

bool Game::recordHash(int depth, int score, int flag, uint64_t key, BitMove move, int real_depth) {
	Hash* hash = &boardHash[key & hash_cutter];

	if(flag == ALPHA && (hash->flag == EXACT || hash->flag == BETA)) {
		return 0;
	}

	if(hash->flag != EMPTY && hash->depth > depth) {
		return 0;
	}

	if(score > WHITE_WIN - 100) {
		score += real_depth;
	} else if(score < -WHITE_WIN + 100) {
		score -= real_depth;
	}

	hash->depth = depth;
	hash->score = score;
	hash->flag = flag;
	hash->key = key;

	if(flag != ALPHA) {
		hash->move = move;
	}
}
<|endoftext|>


<commit_before>#include <cstring>

#include "conf.hh"
#include "io.hh"
#include "Unigrams.hh"
#include "Bigrams.hh"

using namespace std;

int main(int argc, char* argv[]) {

    string vocab_fname;
    string trans_fname;
    int maxlen;
    map<string, flt_type> vocab;
    StringSet *ss_vocab = NULL;
    transitions_t transitions;
    flt_type one_char_min_lp = -500.0;
    bool unigram = true;

    conf::Config config;
    config("usage: segtext [OPTION...] INPUT OUTPUT\n")
      ('h', "help", "", "", "display help")
      ('v', "vocabulary=FILE", "arg", "", "Unigram model file")
      ('t', "transitions=FILE", "arg", "", "Bigram model file")
      ('s', "sentence-mode", "", "", "Prints <s>, </s>, <w> symbols")
      ('8', "utf-8", "", "", "Utf-8 character encoding in use");
    config.default_parse(argc, argv);
    if (config.arguments.size() != 2) config.print_help(stderr, 1);

    bool sentence_mode = config["sentence-mode"].specified;
    bool utf8_encoding = config["utf-8"].specified;
    string in_fname = config.arguments[0];
    string out_fname = config.arguments[1];

    if (!config["vocabulary"].specified && !config["transitions"].specified) {
        cerr << "Please define vocabulary or transitions" << endl;
        exit(0);
    }

    if (config["vocabulary"].specified && config["transitions"].specified) {
        cerr << "Please don't define both vocabulary and transitions" << endl;
        exit(0);
    }

    if (config["vocabulary"].specified) {
        vocab_fname = config["vocabulary"].get_str();
        cerr << "Reading vocabulary " << vocab_fname << endl;
        int retval = Unigrams::read_vocab(vocab_fname, vocab, maxlen, utf8_encoding);
        if (retval < 0) {
            cerr << "something went wrong reading vocabulary" << endl;
            exit(0);
        }
        cerr << "\t" << "size: " << vocab.size() << endl;
        cerr << "\t" << "maximum string length: " << maxlen << endl;
        ss_vocab = new StringSet(vocab);
    }

    if (config["transitions"].specified) {
        unigram = false;
        trans_fname = config["transitions"].get_str();
        cerr << "Reading transitions " << trans_fname << endl;
        int retval = Bigrams::read_transitions(transitions, trans_fname);
        Bigrams::trans_to_vocab(transitions, vocab);
        ss_vocab = new StringSet(vocab);
        if (retval < 0) {
            cerr << "something went wrong reading transitions" << endl;
            exit(0);
        }
        cerr << "\t" << "vocabulary size: " << transitions.size() << endl;
        cerr << "\t" << "transitions: " << retval << endl;
    }

    cerr << "Segmenting corpus" << endl;
    SimpleFileInput infile(in_fname);
    SimpleFileOutput outfile(out_fname);

    string line;
    while (infile.getline(line)) {

        for (unsigned int i=0; i<line.size(); i++) {
            string currchr {line[i]};
            if (!ss_vocab->includes(currchr))
                ss_vocab->add(currchr, one_char_min_lp);
        }

        vector<string> best_path;
        if (unigram)
            viterbi(*ss_vocab, line, best_path, true, utf8_encoding);
        else {
            FactorGraph fg(line, start_end_symbol, *ss_vocab);
            viterbi(transitions, fg, best_path);
            best_path.erase(best_path.begin());
            best_path.erase(best_path.end());
        }

        if (best_path.size() == 0) {
            cerr << "warning, no segmentation for line: " << line << endl;
            continue;
        }

        // Print out the best path
        if (sentence_mode) {
            outfile << "<s> <w>";
            for (unsigned int i=0; i<best_path.size(); i++) {
                if (best_path[i] == " ")
                    outfile << " <w>";
                else
                    outfile << " " << best_path[i];
            }
            outfile << " <w> </s>\n";
        }
        else {
            for (unsigned int i=0; i<best_path.size(); i++) {
                if (best_path[i] == " ")
                    outfile << "\t";
                else {
                    if (i>0) outfile << " ";
                    outfile << best_path[i];
                }
            }
            outfile << "\n";
        }
    }

    outfile.close();
    if (ss_vocab != NULL) delete ss_vocab;
    exit(0);
}

<commit_msg>Improved print options.<commit_after>#include <cstring>

#include "conf.hh"
#include "io.hh"
#include "Unigrams.hh"
#include "Bigrams.hh"

using namespace std;

int main(int argc, char* argv[]) {

    string vocab_fname;
    string trans_fname;
    int maxlen;
    map<string, flt_type> vocab;
    StringSet *ss_vocab = NULL;
    transitions_t transitions;
    flt_type one_char_min_lp = -500.0;
    bool unigram = true;

    conf::Config config;
    config("usage: segtext [OPTION...] INPUT OUTPUT\n")
      ('h', "help", "", "", "display help")
      ('v', "vocabulary=FILE", "arg", "", "Unigram model file")
      ('t', "transitions=FILE", "arg", "", "Bigram model file")
      ('s', "sentence-mode", "", "", "Prints <s> and </s> symbols")
      ('b', "word-boundaries", "", "", "Prints <w> symbols between words")
      ('8', "utf-8", "", "", "Utf-8 character encoding in use");
    config.default_parse(argc, argv);
    if (config.arguments.size() != 2) config.print_help(stderr, 1);

    bool sentence_mode = config["sentence-mode"].specified;
    bool word_boundaries = config["word-boundaries"].specified;
    bool utf8_encoding = config["utf-8"].specified;
    string in_fname = config.arguments[0];
    string out_fname = config.arguments[1];

    if (!config["vocabulary"].specified && !config["transitions"].specified) {
        cerr << "Please define vocabulary or transitions" << endl;
        exit(0);
    }

    if (config["vocabulary"].specified && config["transitions"].specified) {
        cerr << "Please don't define both vocabulary and transitions" << endl;
        exit(0);
    }

    if (!sentence_mode && word_boundaries)
        cerr << "Word boundaries printed only in sentence mode." << endl;

    if (config["vocabulary"].specified) {
        vocab_fname = config["vocabulary"].get_str();
        cerr << "Reading vocabulary " << vocab_fname << endl;
        int retval = Unigrams::read_vocab(vocab_fname, vocab, maxlen, utf8_encoding);
        if (retval < 0) {
            cerr << "something went wrong reading vocabulary" << endl;
            exit(0);
        }
        cerr << "\t" << "size: " << vocab.size() << endl;
        cerr << "\t" << "maximum string length: " << maxlen << endl;
        ss_vocab = new StringSet(vocab);
    }

    if (config["transitions"].specified) {
        unigram = false;
        trans_fname = config["transitions"].get_str();
        cerr << "Reading transitions " << trans_fname << endl;
        int retval = Bigrams::read_transitions(transitions, trans_fname);
        Bigrams::trans_to_vocab(transitions, vocab);
        ss_vocab = new StringSet(vocab);
        if (retval < 0) {
            cerr << "something went wrong reading transitions" << endl;
            exit(0);
        }
        cerr << "\t" << "vocabulary size: " << transitions.size() << endl;
        cerr << "\t" << "transitions: " << retval << endl;
    }

    cerr << "Segmenting corpus" << endl;
    SimpleFileInput infile(in_fname);
    SimpleFileOutput outfile(out_fname);

    string line;
    while (infile.getline(line)) {

        for (unsigned int i=0; i<line.size(); i++) {
            string currchr {line[i]};
            if (!ss_vocab->includes(currchr))
                ss_vocab->add(currchr, one_char_min_lp);
        }

        vector<string> best_path;
        if (unigram)
            viterbi(*ss_vocab, line, best_path, true, utf8_encoding);
        else {
            FactorGraph fg(line, start_end_symbol, *ss_vocab);
            viterbi(transitions, fg, best_path);
            best_path.erase(best_path.begin());
            best_path.erase(best_path.end());
        }

        if (best_path.size() == 0) {
            cerr << "warning, no segmentation for line: " << line << endl;
            continue;
        }

        // Print out the best path
        if (sentence_mode) {
            if (word_boundaries) outfile << "<s> <w>";
            else outfile << "<s>";
            for (unsigned int i=0; i<best_path.size(); i++) {
                if (best_path[i] == " " && word_boundaries)
                    outfile << " <w>";
                else
                    outfile << " " << best_path[i];
            }
            if (word_boundaries) outfile << " <w> </s>\n";
            else outfile << " </s>\n";
        }
        else {
            for (unsigned int i=0; i<best_path.size(); i++) {
                if (best_path[i] == " ")
                    outfile << "\t";
                else {
                    if (i>0) outfile << " ";
                    outfile << best_path[i];
                }
            }
            outfile << "\n";
        }
    }

    outfile.close();
    if (ss_vocab != NULL) delete ss_vocab;
    exit(0);
}

<|endoftext|>


<commit_before>/***************************************************************
 *
 * Copyright (C) 1990-2007, Condor Team, Computer Sciences Department,
 * University of Wisconsin-Madison, WI.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License.  You may
 * obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ***************************************************************/

#include "condor_common.h"
#include "condor_debug.h"
#include "process_control.WINDOWS.h"
#include "ntsysinfo.h"
#include "HashTable.h"
#include "../condor_daemon_core.V6/condor_softkill.h"

static char* soft_kill_binary = NULL;

void
set_windows_soft_kill_binary(char* binary)
{
	if (soft_kill_binary != NULL) {
		free(soft_kill_binary);
	}
	soft_kill_binary = strdup(binary);
	ASSERT(soft_kill_binary != NULL);
}

bool
windows_soft_kill(DWORD pid)
{
	// softkill on Windows is implemented by posting a WM_CLOSE to
	// a top-level window owned by the process (if the process owns
	// more than 1 top-level window, it is undefined which one will
	// get the WM_CLOSE). finding the window and posting the message
	// are done using the condor_softkill tool
	//

	// bail out if the binary wasn't set
	//
	if (soft_kill_binary == NULL) {
		dprintf(D_ALWAYS,
		        "windows_soft_kill error: softkill binary not set\n");
		return false;
	}

	// construct the command line for the call to CreateProcess
	//
	char command_line[1024];
	command_line[1023] = '\0';
	int ret = snprintf(command_line,
	                   1024,
	                   "\"%s\" %u debug.txt",
	                   soft_kill_binary,
					   pid);
	if (ret < 0) {
		dprintf(D_ALWAYS,
		        "windows:soft_kill: snprintf error: %s (%d)\n",
		        strerror(errno),
		        errno);
		return false;
	}
	if (command_line[1023] != '\0') {
		dprintf(D_ALWAYS,
		        "windows_soft_kill: command line too long\n");
		return false;
	}

	// call CreateProcess to run the softkill executable
	//
	STARTUPINFO si;
	memset(&si, 0, sizeof(STARTUPINFO));
	PROCESS_INFORMATION pi;
	if (CreateProcess(NULL,         // get the executable from the command line
	                  command_line,
	                  NULL,         // default process security
	                  NULL,         // default security for primary thread
	                  FALSE,        // don't inherit handles
	                  0,            // no flags
	                  NULL,         // inherit our environment
	                  NULL,         // inherit our working directory
	                  &si,
	                  &pi) == FALSE)
	{
		dprintf(D_ALWAYS,
		        "windows_soft_kill: CreateProcess error: %u\n",
		        GetLastError());
		return false;
	}
	CloseHandle(pi.hThread);

	// wait for the exit code; fail if we can't get it or if
	// it indicates failure
	//
	if (WaitForSingleObject(pi.hProcess, INFINITE) != WAIT_OBJECT_0) {
		dprintf(D_ALWAYS,
		        "windows_soft_kill: WaitForSingleObject error: %u\n",
		        GetLastError());
	}
	DWORD exit_code;
	BOOL result = GetExitCodeProcess(pi.hProcess, &exit_code);
	DWORD error = GetLastError();
	CloseHandle(pi.hProcess);
	if (result == FALSE) {
		dprintf(D_ALWAYS,
		        "windows_soft_kill: GetExitCodeProcess error: %u\n",
		        GetLastError());
		return false;
	}
	if (exit_code != SOFTKILL_SUCCESS) {
		dprintf(D_ALWAYS,
		        "windows_soft_kill: error code from condor_softkill: %u\n",
		        exit_code);
		return false;
	}

	// i guess it worked
	//
	return true;
}

bool
windows_hard_kill(DWORD pid)
{
	HANDLE process = OpenProcess(PROCESS_TERMINATE, FALSE, pid);
	if (process == NULL) {
		dprintf(D_ALWAYS,
		        "windows_hard_kill: OpenProcess error: %u\n",
		        GetLastError());
		return false;
	}
	if (TerminateProcess(process, 0) == FALSE) {
		dprintf(D_ALWAYS,
		        "windows_hard_kill: TerminateProcess error: %u\n",
		        GetLastError());
		return false;
	}
	CloseHandle(process);

	return true;
}

// stupid hash function for DWORD (need by the hash tables
// used in the suspend/contiunue code below)
//
static unsigned int
hash_func(const DWORD& n)
{
	return n;
}

// stupid object for getting the list of threads for a process
//
static CSysinfo sys_info;

// record to keep track of the threads in a suspended process
//
struct SuspendedThread {

	// cache a handle to each thread while the process is suspended
	//
	HANDLE handle;

	// how many suspends did we send this thread? (so we know how
	// many resumes to send!)
	//
	int num_suspends;
};

// for each suspended process we keep a hash table of its
// SuspendedThread structures (keyed on thread id)
//
typedef HashTable<DWORD, SuspendedThread*> SuspendedProcess;

// we keep track of all suspended processes via a hash table
// keyed on process id
//
static HashTable<DWORD, SuspendedProcess*> suspended_processes(11, hash_func);

bool
windows_suspend(DWORD pid)
{
	// make sure we haven't already suspended this process, then allocate
	// a SuspendedProcess object and add it to the hash table
	//
	SuspendedProcess* sp;
	int ret = suspended_processes.lookup(pid, sp);
	ASSERT(ret == -1);
	sp = new SuspendedProcess(11, hash_func);
	ASSERT(sp != NULL);
	ret = suspended_processes.insert(pid, sp);
	ASSERT(ret != -1);

	// we need to loop until iterating through the thread list for this
	// process shows that all threads are suspended
	//
	bool finished = false;
	while (!finished) {

		// assume we're going to succeed on this iteration until
		// proven otherwise
		//
		finished = true;

		// get the list of threads for this process
		//
		ExtArray<DWORD> tid_array;
		int num_tids = sys_info.GetTIDs(pid, tid_array);
		if (num_tids == 0) {
			// TODO: we need to handle this case!!!
			//
			EXCEPT("windows_suspend_process failed: can't get threads for pid %u",
			       pid);
		}

		// go through the thread list, calling SuspendThread on each
		//
		for (int i = 0; i < num_tids; i++) {

			// see if we already have a record for this thread
			//
			SuspendedThread* st;
			ret = sp->lookup(tid_array[i], st);
			if (ret == -1) {

				// no record yet; open up this thread's handle
				//
				HANDLE handle = OpenThread(THREAD_SUSPEND_RESUME, FALSE, tid_array[i]);
				if (handle == NULL) {

					// the thread probably exited; however, it may have called ResumeThread
					// or CreateThread first, so we'll have to loop again
					//
					finished = false;
					dprintf(D_FULLDEBUG,
					        "windows_suspend_process: OpenThread error: %u\n",
					        GetLastError());
					continue;
				}

				// now that we have a handle, create a record for this thread
				//
				st = new SuspendedThread;
				ASSERT(st != NULL);
				st->handle = handle;
				st->num_suspends = 0;
			}

			// now suspend the thread
			//
			DWORD result = SuspendThread(st->handle);
			if (result == (DWORD)-1) {

				// how can this happen? maybe the thread exited?
				//
				finished = false;
				dprintf(D_FULLDEBUG,
						"windows_suspend_process: SuspendThread error: %u\n",
						GetLastError());
				continue;
			}

			// if the thread was not suspended prior to our SuspendThread call,
			// it may have called CreateThread or ResumeThread so we need to loop
			// at least one more time
			//
			if (result == 0) {
				finished = false;
			}

			// keep track of how many suspends we've sent this thread
			//
			st->num_suspends++;
		}
	}

	return true;
}

bool
windows_continue(DWORD pid)
{
	// we should have a record from when we suspended this process
	//
	SuspendedProcess* sp;
	int ret = suspended_processes.lookup(pid, sp);
	ASSERT(ret != -1);

	SuspendedThread* st;
	sp->startIterations();
	while (sp->iterate(st)) {

		// call ResumeThread as many times as we called SuspendThread
		// for this thread above
		//
		for (int i = 0; i < st->num_suspends; i++) {
			if (ResumeThread(st->handle) == (DWORD)-1) {
				dprintf(D_FULLDEBUG,
				        "windows_continue_process: ResumeThread error: %u\n",
				        GetLastError());
			}
		}

		// close our handle to this thread
		//
		if (CloseHandle(st->handle) == FALSE) {
			dprintf(D_FULLDEBUG,
			        "windows_continue_process: CloseHandle error: %u\n",
			        GetLastError());
		}

		// clean up this SuspendedThread object
		//
		delete st;
	}

	// clean up the SuspendedProcess object and remove it from the
	// hash table of processes
	//
	delete sp;
	ret = suspended_processes.remove(pid);
	ASSERT(ret != -1);

	return true;
}
<commit_msg>Fix job suspend / resume on windows<commit_after>/***************************************************************
 *
 * Copyright (C) 1990-2007, Condor Team, Computer Sciences Department,
 * University of Wisconsin-Madison, WI.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License.  You may
 * obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ***************************************************************/

#include "condor_common.h"
#include "condor_debug.h"
#include "process_control.WINDOWS.h"
#include "ntsysinfo.h"
#include "HashTable.h"
#include "../condor_daemon_core.V6/condor_softkill.h"

static char* soft_kill_binary = NULL;

void
set_windows_soft_kill_binary(char* binary)
{
	if (soft_kill_binary != NULL) {
		free(soft_kill_binary);
	}
	soft_kill_binary = strdup(binary);
	ASSERT(soft_kill_binary != NULL);
}

bool
windows_soft_kill(DWORD pid)
{
	// softkill on Windows is implemented by posting a WM_CLOSE to
	// a top-level window owned by the process (if the process owns
	// more than 1 top-level window, it is undefined which one will
	// get the WM_CLOSE). finding the window and posting the message
	// are done using the condor_softkill tool
	//

	// bail out if the binary wasn't set
	//
	if (soft_kill_binary == NULL) {
		dprintf(D_ALWAYS,
		        "windows_soft_kill error: softkill binary not set\n");
		return false;
	}

	// construct the command line for the call to CreateProcess
	//
	char command_line[1024];
	command_line[1023] = '\0';
	int ret = snprintf(command_line,
	                   1024,
	                   "\"%s\" %u debug.txt",
	                   soft_kill_binary,
					   pid);
	if (ret < 0) {
		dprintf(D_ALWAYS,
		        "windows:soft_kill: snprintf error: %s (%d)\n",
		        strerror(errno),
		        errno);
		return false;
	}
	if (command_line[1023] != '\0') {
		dprintf(D_ALWAYS,
		        "windows_soft_kill: command line too long\n");
		return false;
	}

	// call CreateProcess to run the softkill executable
	//
	STARTUPINFO si;
	memset(&si, 0, sizeof(STARTUPINFO));
	PROCESS_INFORMATION pi;
	if (CreateProcess(NULL,         // get the executable from the command line
	                  command_line,
	                  NULL,         // default process security
	                  NULL,         // default security for primary thread
	                  FALSE,        // don't inherit handles
	                  0,            // no flags
	                  NULL,         // inherit our environment
	                  NULL,         // inherit our working directory
	                  &si,
	                  &pi) == FALSE)
	{
		dprintf(D_ALWAYS,
		        "windows_soft_kill: CreateProcess error: %u\n",
		        GetLastError());
		return false;
	}
	CloseHandle(pi.hThread);

	// wait for the exit code; fail if we can't get it or if
	// it indicates failure
	//
	if (WaitForSingleObject(pi.hProcess, INFINITE) != WAIT_OBJECT_0) {
		dprintf(D_ALWAYS,
		        "windows_soft_kill: WaitForSingleObject error: %u\n",
		        GetLastError());
	}
	DWORD exit_code;
	BOOL result = GetExitCodeProcess(pi.hProcess, &exit_code);
	DWORD error = GetLastError();
	CloseHandle(pi.hProcess);
	if (result == FALSE) {
		dprintf(D_ALWAYS,
		        "windows_soft_kill: GetExitCodeProcess error: %u\n",
		        GetLastError());
		return false;
	}
	if (exit_code != SOFTKILL_SUCCESS) {
		dprintf(D_ALWAYS,
		        "windows_soft_kill: error code from condor_softkill: %u\n",
		        exit_code);
		return false;
	}

	// i guess it worked
	//
	return true;
}

bool
windows_hard_kill(DWORD pid)
{
	HANDLE process = OpenProcess(PROCESS_TERMINATE, FALSE, pid);
	if (process == NULL) {
		dprintf(D_ALWAYS,
		        "windows_hard_kill: OpenProcess error: %u\n",
		        GetLastError());
		return false;
	}
	if (TerminateProcess(process, 0) == FALSE) {
		dprintf(D_ALWAYS,
		        "windows_hard_kill: TerminateProcess error: %u\n",
		        GetLastError());
		return false;
	}
	CloseHandle(process);

	return true;
}

// stupid hash function for DWORD (need by the hash tables
// used in the suspend/contiunue code below)
//
static unsigned int
hash_func(const DWORD& n)
{
	return n;
}

// stupid object for getting the list of threads for a process
//
static CSysinfo sys_info;

// record to keep track of the threads in a suspended process
//
struct SuspendedThread {

	// cache a handle to each thread while the process is suspended
	//
	HANDLE handle;

	// how many suspends did we send this thread? (so we know how
	// many resumes to send!)
	//
	int num_suspends;
};

// for each suspended process we keep a hash table of its
// SuspendedThread structures (keyed on thread id)
//
typedef HashTable<DWORD, SuspendedThread*> SuspendedProcess;

// we keep track of all suspended processes via a hash table
// keyed on process id
//
static HashTable<DWORD, SuspendedProcess*> suspended_processes(11, hash_func);

bool
windows_suspend(DWORD pid)
{
	// make sure we haven't already suspended this process, then allocate
	// a SuspendedProcess object and add it to the hash table
	//
	SuspendedProcess* sp;
	int ret = suspended_processes.lookup(pid, sp);
	ASSERT(ret == -1);
	sp = new SuspendedProcess(11, hash_func);
	ASSERT(sp != NULL);
	ret = suspended_processes.insert(pid, sp);
	ASSERT(ret != -1);

	// we need to loop until iterating through the thread list for this
	// process shows that all threads are suspended
	//
	bool finished = false;
	while (!finished) {

		// assume we're going to succeed on this iteration until
		// proven otherwise
		//
		finished = true;

		// get the list of threads for this process
		//
		ExtArray<DWORD> tid_array;
		int num_tids = sys_info.GetTIDs(pid, tid_array);
		if (num_tids == 0) {
			// TODO: we need to handle this case!!!
			//
			EXCEPT("windows_suspend_process failed: can't get threads for pid %u",
			       pid);
		}

		// go through the thread list, calling SuspendThread on each
		//
		for (int i = 0; i < num_tids; i++) {

			// see if we already have a record for this thread
			//
			SuspendedThread* st;
			ret = sp->lookup(tid_array[i], st);
			if (ret == -1) {

				// no record yet; open up this thread's handle
				//
				HANDLE handle = OpenThread(THREAD_SUSPEND_RESUME, FALSE, tid_array[i]);
				if (handle == NULL) {

					// the thread probably exited; however, it may have called ResumeThread
					// or CreateThread first, so we'll have to loop again
					//
					finished = false;
					dprintf(D_FULLDEBUG,
					        "windows_suspend_process: OpenThread error: %u\n",
					        GetLastError());
					continue;
				}

				// now that we have a handle, create a record for this thread
				//
				st = new SuspendedThread;
				ASSERT(st != NULL);
				st->handle = handle;
				st->num_suspends = 0;
				ret = sp->insert(tid_array[i], st);
				ASSERT(ret != -1);
			}

			// now suspend the thread
			//
			DWORD result = SuspendThread(st->handle);
			if (result == (DWORD)-1) {

				// how can this happen? maybe the thread exited?
				//
				finished = false;
				dprintf(D_FULLDEBUG,
						"windows_suspend_process: SuspendThread error: %u\n",
						GetLastError());
				continue;
			}

			// if the thread was not suspended prior to our SuspendThread call,
			// it may have called CreateThread or ResumeThread so we need to loop
			// at least one more time
			//
			if (result == 0) {
				finished = false;
			}

			// keep track of how many suspends we've sent this thread
			//
			st->num_suspends++;
		}
	}

	return true;
}

bool
windows_continue(DWORD pid)
{
	// we should have a record from when we suspended this process
	//
	SuspendedProcess* sp;
	int ret = suspended_processes.lookup(pid, sp);
	ASSERT(ret != -1);

	SuspendedThread* st;
	sp->startIterations();
	while (sp->iterate(st)) {

		// call ResumeThread as many times as we called SuspendThread
		// for this thread above
		//
		for (int i = 0; i < st->num_suspends; i++) {
			if (ResumeThread(st->handle) == (DWORD)-1) {
				dprintf(D_FULLDEBUG,
				        "windows_continue_process: ResumeThread error: %u\n",
				        GetLastError());
			}
		}

		// close our handle to this thread
		//
		if (CloseHandle(st->handle) == FALSE) {
			dprintf(D_FULLDEBUG,
			        "windows_continue_process: CloseHandle error: %u\n",
			        GetLastError());
		}

		// clean up this SuspendedThread object
		//
		delete st;
	}

	// clean up the SuspendedProcess object and remove it from the
	// hash table of processes
	//
	delete sp;
	ret = suspended_processes.remove(pid);
	ASSERT(ret != -1);

	return true;
}
<|endoftext|>


<commit_before>/* Copyright (C) 2005, 2006, 2007, 2008
 * Robert Geisberger, Dominik Schultes, Peter Sanders,
 * Universitaet Karlsruhe (TH)
 *
 * This file is part of Contraction Hierarchies.
 *
 * Contraction Hierarchies is free software; you can redistribute it
 * and/or modify it under the terms of the GNU Affero General Public License
 * as published by the Free Software Foundation; either version 3 of
 * the License, or (at your option) any later version.
 *
 * Contraction Hierarchies is distributed in the hope that it will be
 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty
 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with Contraction Hierarchies; see the file COPYING; if not,
 * see <http://www.gnu.org/licenses/>.
 */

#include <iostream>
#include <iomanip>
#include <fstream>
#include <sys/types.h>
#include <fcntl.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl.h>

using namespace std;
using namespace google::protobuf::io;

/**
 * Performing the bucket scans is a crucial part of the many-to-many computation.
 * It can be switched off to allow time measurements of the forward search
 * without accounting for the bucket scans.
 */
const bool performBucketScans = true;

#include "../config.h"
#include "../stats/utils.h"
Counter counter;
//#include "../datastr/general.h"
#include "../datastr/graph/graph.h"
#include "../datastr/graph/SearchGraph.h"

typedef datastr::graph::SearchGraph TransitGraph;

//#include "../stats/log.h"
#include "../processing/DijkstraCH.h"
#include "manyToMany.h"
#include "protos/ch/many2many.pb.h"

typedef datastr::graph::SearchGraph MyGraph;
typedef DijkstraCHManyToManyFW DijkstraManyToManyFW;
typedef DijkstraCHManyToManyBW DijkstraManyToManyBW;

static void readNodes(const MyGraph *const g, const google::protobuf::RepeatedField<uint32_t>& in, vector<NodeID>& vs) {
    vs.resize(in.size());
    for (auto i=0; i < in.size(); i++) {
        // map the actual node ID to the node ID that is used internally by
        // highway-node routing
        vs[i] = g->mapExtToIntNodeID(in.Get(i));
    }
}

static void readMessage(const string& filename, ::google::protobuf::Message& message) {
    int fd = open(filename.c_str(), O_RDONLY);
    FileInputStream raw_input(fd);
    raw_input.SetCloseOnDelete(true);
    message.ParseFromZeroCopyStream(&raw_input);
}

static void writeMessage(const string& filename, ::google::protobuf::Message& message) {
    int fd = open(filename.c_str(), O_WRONLY);
    FileOutputStream raw_output(fd);
    raw_output.SetCloseOnDelete(true);
    message.SerializeToZeroCopyStream(&raw_output);
}

MyGraph *const readGraph(const string& inGraphFn) {
    ifstream inGraph(inGraphFn, ios::binary);
    if (!inGraph) {
        cerr << "Input file '" << inGraphFn << "' couldn't be opened." << endl;
        exit(-1);
    }
    return new MyGraph(inGraph);
}

/** The main program. */
int main(int argc, char *argv[])
{
    // read arguments
    if (argc < 2) {
        cerr << "Too few arguments!" << endl
             << "Usage: ./manyToMany <inSgrGraph> <inNodes> <outMatrix>" << endl;
        exit(-1);
    }

    const string inGraphFn   = argv[1];
    const string inNodesFn   = argv[2];
    const string outMatrixFn = argv[3];

    const bool writeSourceNodes = false;
    const bool writeMatrix = false;
    const bool validateResult = false;

    VERBOSE( cerr << "read graph from '" << inGraphFn << "'" << endl);
    VERBOSE( cerr << "read nodes from '" << inNodesFn << "'" << endl);
    VERBOSE( cerr << "write matrix to '" << outMatrixFn << "'" << endl);
    VERBOSE( if (writeSourceNodes) cerr << "write source nodes to cerr" << endl );
    VERBOSE( if (writeMatrix)      cerr << "write matrix to cerr" << endl );
    VERBOSE( if (validateResult)   cerr << "validate result" << endl );

    uint32_t sz_sources, sz_targets;
    cin >> sz_sources >> sz_targets;

    // read graph
    MyGraph *const graph = readGraph(inGraphFn);

    VERBOSE( cerr << "Graph read." << endl );

    // read the input
    protos::ch::Nodes nodes;
    readMessage(inNodesFn, nodes);

    vector<NodeID> sources;
    readNodes(graph, nodes.sources(), sources);

    vector<NodeID> targets;
    readNodes(graph, nodes.targets(), targets);

    // create many-to-many object
    LevelID earlyStopLevel = 10;
    ManyToMany<MyGraph, DijkstraManyToManyFW, DijkstraManyToManyBW, performBucketScans> mtm(graph, earlyStopLevel);

    // compute matrix
    Matrix<EdgeWeight> matrix(sources.size(), targets.size());
    mtm.computeMatrix( sources, targets, matrix );

    // generate and write output
    protos::ch::Matrix outMatrix;
    for (int i=0; i < sources.size(); i++) {
        protos::ch::Row* row = outMatrix.add_rows();
        for (int j=0; j < targets.size(); j++) {
            row->add_data(matrix.value(i, j));
        }
    }
    writeMessage(outMatrixFn, outMatrix);

    if (validateResult) {
        // compute reference solution
        Matrix<EdgeWeight> matrixRef(sources.size(), targets.size());
        mtm.computeMatrixNaive( sources, targets, matrixRef);

        // check
        if (matrix == matrixRef) {
            VERBOSE( cerr << "Solution validated." << endl );
        } else {
            cerr << "Wrong solution!" << endl;
        }
    }

    if (writeMatrix) {
        VERBOSE( cerr << "write matrix" << endl );
        cerr << matrix;
    }

    delete graph;
}

// doesn't look nice, but required by the compiler (gcc 4)
const EdgeWeight Weight::MAX_VALUE;
<commit_msg>Tweaks in I/O.<commit_after>/* Copyright (C) 2005, 2006, 2007, 2008
 * Robert Geisberger, Dominik Schultes, Peter Sanders,
 * Universitaet Karlsruhe (TH)
 *
 * This file is part of Contraction Hierarchies.
 *
 * Contraction Hierarchies is free software; you can redistribute it
 * and/or modify it under the terms of the GNU Affero General Public License
 * as published by the Free Software Foundation; either version 3 of
 * the License, or (at your option) any later version.
 *
 * Contraction Hierarchies is distributed in the hope that it will be
 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty
 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with Contraction Hierarchies; see the file COPYING; if not,
 * see <http://www.gnu.org/licenses/>.
 */

#include <iostream>
#include <iomanip>
#include <fstream>
#include <sys/types.h>
#include <fcntl.h>
#include <unistd.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl.h>

using namespace std;
using namespace google::protobuf::io;

/**
 * Performing the bucket scans is a crucial part of the many-to-many computation.
 * It can be switched off to allow time measurements of the forward search
 * without accounting for the bucket scans.
 */
const bool performBucketScans = true;

#include "../config.h"
#include "../stats/utils.h"
Counter counter;
//#include "../datastr/general.h"
#include "../datastr/graph/graph.h"
#include "../datastr/graph/SearchGraph.h"

typedef datastr::graph::SearchGraph TransitGraph;

//#include "../stats/log.h"
#include "../processing/DijkstraCH.h"
#include "manyToMany.h"
#include "protos/ch/many2many.pb.h"

typedef datastr::graph::SearchGraph MyGraph;
typedef DijkstraCHManyToManyFW DijkstraManyToManyFW;
typedef DijkstraCHManyToManyBW DijkstraManyToManyBW;

static void readNodes(const MyGraph *const g, const google::protobuf::RepeatedField<uint32_t>& in, vector<NodeID>& vs) {
    vs.resize(in.size());
    for (auto i=0; i < in.size(); i++) {
        // map the actual node ID to the node ID that is used internally by
        // highway-node routing
        vs[i] = g->mapExtToIntNodeID(in.Get(i));
    }
}

static void readMessage(const string& filename, ::google::protobuf::Message& message) {
    int fd = open(filename.c_str(), O_RDONLY);
    message.ParseFromFileDescriptor(fd);
    close(fd);
}

static void writeMessage(const string& filename, ::google::protobuf::Message& message) {
    int fd = creat(filename.c_str(), 0644);
    message.SerializeToFileDescriptor(fd);
    close(fd);
}

MyGraph *const readGraph(const string& inGraphFn) {
    ifstream inGraph(inGraphFn, ios::binary);
    if (!inGraph) {
        cerr << "Input file '" << inGraphFn << "' couldn't be opened." << endl;
        exit(-1);
    }
    return new MyGraph(inGraph);
}

/** The main program. */
int main(int argc, char *argv[])
{
    // read arguments
    if (argc < 2) {
        cerr << "Too few arguments!" << endl
             << "Usage: ./manyToMany <inSgrGraph> <inNodes> <outMatrix>" << endl;
        exit(-1);
    }

    const string inGraphFn   = argv[1];
    const string inNodesFn   = argv[2];
    const string outMatrixFn = argv[3];

    const bool writeSourceNodes = false;
    const bool writeMatrix = false;
    const bool validateResult = false;

    VERBOSE( cerr << "read graph from '" << inGraphFn << "'" << endl);
    VERBOSE( cerr << "read nodes from '" << inNodesFn << "'" << endl);
    VERBOSE( cerr << "write matrix to '" << outMatrixFn << "'" << endl);
    VERBOSE( if (writeSourceNodes) cerr << "write source nodes to cerr" << endl );
    VERBOSE( if (writeMatrix)      cerr << "write matrix to cerr" << endl );
    VERBOSE( if (validateResult)   cerr << "validate result" << endl );

    // read graph
    MyGraph *const graph = readGraph(inGraphFn);

    VERBOSE( cerr << "Graph read." << endl );

    // read the input
    protos::ch::Nodes nodes;
    readMessage(inNodesFn, nodes);

    vector<NodeID> sources;
    readNodes(graph, nodes.sources(), sources);

    vector<NodeID> targets;
    readNodes(graph, nodes.targets(), targets);

    // create many-to-many object
    LevelID earlyStopLevel = 10;
    ManyToMany<MyGraph, DijkstraManyToManyFW, DijkstraManyToManyBW, performBucketScans> mtm(graph, earlyStopLevel);

    // compute matrix
    Matrix<EdgeWeight> matrix(sources.size(), targets.size());
    mtm.computeMatrix( sources, targets, matrix );

    // generate and write output
    protos::ch::Matrix outMatrix;
    for (int i=0; i < sources.size(); i++) {
        protos::ch::Row* row = outMatrix.add_rows();
        for (int j=0; j < targets.size(); j++) {
            row->add_data(matrix.value(i, j));
        }
    }
    writeMessage(outMatrixFn, outMatrix);

    if (validateResult) {
        // compute reference solution
        Matrix<EdgeWeight> matrixRef(sources.size(), targets.size());
        mtm.computeMatrixNaive( sources, targets, matrixRef);

        // check
        if (matrix == matrixRef) {
            VERBOSE( cerr << "Solution validated." << endl );
        } else {
            cerr << "Wrong solution!" << endl;
        }
    }

    if (writeMatrix) {
        VERBOSE( cerr << "write matrix" << endl );
        cerr << matrix;
    }

    delete graph;
}

// doesn't look nice, but required by the compiler (gcc 4)
const EdgeWeight Weight::MAX_VALUE;
<|endoftext|>


<commit_before>/* Copyright (c) 2015-present Advanced Micro Devices, Inc.

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
 in the Software without restriction, including without limitation the rights
 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:

 The above copyright notice and this permission notice shall be included in
 all copies or substantial portions of the Software.

 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE. */

#ifndef HIP_SRC_HIP_INTERNAL_H
#define HIP_SRC_HIP_INTERNAL_H

#include "vdi_common.hpp"
#include "hip_prof_api.h"
#include "trace_helper.h"
#include "utils/debug.hpp"
#include "hip_formatting.hpp"
#include <unordered_set>
#include <thread>
#include <stack>
#include <mutex>
#include <iterator>
#ifdef _WIN32
#include <process.h>
#else
#include <unistd.h>
#endif

/*! IHIP IPC MEMORY Structure */
#define IHIP_IPC_MEM_HANDLE_SIZE   32
#define IHIP_IPC_MEM_RESERVED_SIZE LP64_SWITCH(28,24)

typedef struct ihipIpcMemHandle_st {
  char ipc_handle[IHIP_IPC_MEM_HANDLE_SIZE];  ///< ipc memory handle on ROCr
  size_t psize;
  char reserved[IHIP_IPC_MEM_RESERVED_SIZE];
} ihipIpcMemHandle_t;

#ifdef _WIN32
  inline int getpid() { return _getpid(); }
#endif

#define HIP_INIT() \
  std::call_once(hip::g_ihipInitialized, hip::init);       \
  if (hip::g_device == nullptr && g_devices.size() > 0) {  \
    hip::g_device = g_devices[0];                          \
  }

// This macro should be called at the beginning of every HIP API.
#define HIP_INIT_API(cid, ...)                               \
  ClPrint(amd::LOG_INFO, amd::LOG_API, "%-5d: [%zx] %s ( %s )", getpid(), std::this_thread::get_id(), __func__, ToString( __VA_ARGS__ ).c_str()); \
  amd::Thread* thread = amd::Thread::current();              \
  if (!VDI_CHECK_THREAD(thread)) {                           \
    HIP_RETURN(hipErrorOutOfMemory);                         \
  }                                                          \
  HIP_INIT()                                                 \
  HIP_CB_SPAWNER_OBJECT(cid);

#define HIP_RETURN(ret)          \
  hip::g_lastError = ret;  \
  ClPrint(amd::LOG_INFO, amd::LOG_API, "%-5d: [%zx] %s: Returned %s", getpid(), std::this_thread::get_id(), __func__, hipGetErrorName(hip::g_lastError)); \
  return hip::g_lastError;

namespace hc {
class accelerator;
class accelerator_view;
};

namespace hip {
  class Device;

  class Stream {
    amd::HostQueue* queue_;
    mutable amd::Monitor lock_;
    Device* device_;
    amd::CommandQueue::Priority priority_;
    unsigned int flags_;
    bool null_;

  public:
    Stream(Device* dev, amd::CommandQueue::Priority p, unsigned int f = 0, bool null_stream = false);
    bool Create();
    amd::HostQueue* asHostQueue();
    void Destroy();
    void Finish() const;
    /// Get device ID associated with the current stream;
    int DeviceId() const;
    /// Returns if stream is null stream
    bool Null() const { return null_; }
    /// Returns the lock object for the current stream
    amd::Monitor& Lock() const { return lock_; }
    /// Returns the creation flags for the current stream
    unsigned int Flags() const { return flags_; }
  };

  /// HIP Device class
  class Device {
    amd::Monitor lock_{"Device lock"};
    /// VDI context
    amd::Context* context_;
    /// Device's ID
    /// Store it here so we don't have to loop through the device list every time
    int deviceId_;
    /// VDI host queue for default streams
    Stream null_stream_;
    //Maintain list of user enabled peers
    std::list<int> userEnabledPeers;

  public:
    Device(amd::Context* ctx, int devId):
      context_(ctx), deviceId_(devId), null_stream_(this, amd::CommandQueue::Priority::Normal, 0, true)
        { assert(ctx != nullptr); }
    ~Device() {}

    amd::Context* asContext() const { return context_; }
    int deviceId() const { return deviceId_; }
    void retain() const { context_->retain(); }
    void release() const { context_->release(); }
    const std::vector<amd::Device*>& devices() const { return context_->devices(); }
    hipError_t EnablePeerAccess(int peerDeviceId){
      amd::ScopedLock lock(lock_);
      bool found = (std::find(userEnabledPeers.begin(), userEnabledPeers.end(), peerDeviceId) != userEnabledPeers.end());
      if (found) {
        return hipErrorPeerAccessAlreadyEnabled;
      }
      userEnabledPeers.push_back(peerDeviceId);
      return hipSuccess;
    }
    hipError_t DisablePeerAccess(int peerDeviceId) {
      amd::ScopedLock lock(lock_);
      bool found = (std::find(userEnabledPeers.begin(), userEnabledPeers.end(), peerDeviceId) != userEnabledPeers.end());
      if (found) {
        userEnabledPeers.remove(peerDeviceId);
        return hipSuccess;
      } else {
        return hipErrorPeerAccessNotEnabled;
      }
    }
    amd::HostQueue* NullStream();
  };

  extern std::once_flag g_ihipInitialized;
  /// Current thread's device
  extern thread_local Device* g_device;
  extern thread_local hipError_t g_lastError;
  /// Device representing the host - for pinned memory
  extern Device* host_device;

  extern void init();

  extern Device* getCurrentDevice();

  extern void setCurrentDevice(unsigned int index);

  /// Get VDI queue associated with hipStream
  /// Note: This follows the CUDA spec to sync with default streams
  ///       and Blocking streams
  extern amd::HostQueue* getQueue(hipStream_t s);
  /// Get default stream associated with the VDI context
  extern amd::HostQueue* getNullStream(amd::Context&);
  /// Get default stream of the thread
  extern amd::HostQueue* getNullStream();

  struct Function {
    amd::Kernel* function_;
    amd::Monitor lock_;

    Function(amd::Kernel* f) : function_(f), lock_("function lock") {}
    hipFunction_t asHipFunction() { return reinterpret_cast<hipFunction_t>(this); }

    static Function* asFunction(hipFunction_t f) { return reinterpret_cast<Function*>(f); }
  };

};

struct ihipExec_t {
  dim3 gridDim_;
  dim3 blockDim_;
  size_t sharedMem_;
  hipStream_t hStream_;
  std::vector<char> arguments_;
};

class PlatformState {
  amd::Monitor lock_{"Guards global function map", true};

  std::unordered_map<const void*, std::vector<std::pair<hipModule_t, bool>>> modules_;
  bool initialized_{false};

  void digestFatBinary(const void* data, std::vector<std::pair<hipModule_t, bool>>& programs);
public:
  void init();
  std::vector<std::pair<hipModule_t, bool>>* addFatBinary(const void*data)
  {
    if (initialized_) {
      digestFatBinary(data, modules_[data]);
    }
    return &modules_[data];
  }
  void removeFatBinary(std::vector<std::pair<hipModule_t, bool>>* module)
  {
    for (auto& mod : modules_) {
      if (&mod.second == module) {
        modules_.erase(&mod);
        return;
      }
    }
  }

  struct RegisteredVar {
  public:
    RegisteredVar(): size_(0), devicePtr_(nullptr), amd_mem_obj_(nullptr) {}
    ~RegisteredVar() {}

    hipDeviceptr_t getdeviceptr() const { return devicePtr_; };
    size_t getvarsize() const { return size_; };

    size_t size_;               // Size of the variable
    hipDeviceptr_t devicePtr_;  //Device Memory Address of the variable.
    amd::Memory* amd_mem_obj_;
  };

  struct DeviceFunction {
    std::string deviceName;
    std::vector< std::pair< hipModule_t, bool > >* modules;
    std::vector<hipFunction_t> functions;
  };
  enum DeviceVarKind {
    DVK_Variable,
    DVK_Surface,
    DVK_Texture
  };
  struct DeviceVar {
    DeviceVarKind kind;
    void* shadowVptr;
    std::string hostVar;
    size_t size;
    std::vector< std::pair< hipModule_t, bool > >* modules;
    std::vector<RegisteredVar> rvars;
    bool dyn_undef;
    int type; // surface/texture type
    int norm; // texture has normalized output
    bool shadowAllocated = false; // shadow ptr is allocated on-demand and needs freeing.
  };
private:
  class Module {
  public:
    Module(hipModule_t hip_module_) : hip_module(hip_module_) {}
    std::unordered_map<std::string, DeviceFunction > functions_;
  private:
    hipModule_t hip_module;
  };
  std::unordered_map<hipModule_t, Module*> module_map_;

  std::unordered_map<const void*, DeviceFunction > functions_;
  std::unordered_multimap<std::string, DeviceVar > vars_;
  // Map from the host shadow symbol to its device name. As different modules
  // may have the same name, each symbol is uniquely identified by a pair of
  // module handle and its name.
  std::unordered_map<const void*,
                     std::pair<hipModule_t, std::string>> symbols_;

  static PlatformState* platform_;

  PlatformState() {}
  ~PlatformState() {}
public:
  static PlatformState& instance() {
    if (platform_ == nullptr) {
       // __hipRegisterFatBinary() will call this when app starts, thus
       // there is no multiple entry issue here.
       platform_ =  new PlatformState();
    }
    return *platform_;
  }

  bool unregisterFunc(hipModule_t hmod);
  std::vector< std::pair<hipModule_t, bool> >* unregisterVar(hipModule_t hmod);

  bool findSymbol(const void *hostVar, hipModule_t &hmod, std::string &devName);
  PlatformState::DeviceVar* findVar(std::string hostVar, int deviceId, hipModule_t hmod);
  void registerVarSym(const void *hostVar, hipModule_t hmod, const char *symbolName);
  void registerVar(const char* symbolName, const DeviceVar& var);
  void registerFunction(const void* hostFunction, const DeviceFunction& func);

  bool registerModFuncs(std::vector<std::string>& func_names, hipModule_t* module);
  bool findModFunc(hipFunction_t* hfunc, hipModule_t hmod, const char* name);
  bool createFunc(hipFunction_t* hfunc, hipModule_t hmod, const char* name);
  hipFunction_t getFunc(const void* hostFunction, int deviceId);
  bool getFuncAttr(const void* hostFunction, hipFuncAttributes* func_attr);
  bool getGlobalVar(const char* hostVar, int deviceId, hipModule_t hmod,
                    hipDeviceptr_t* dev_ptr, size_t* size_ptr);
  bool getTexRef(const char* hostVar, hipModule_t hmod, textureReference** texRef);

  bool getGlobalVarFromSymbol(const void* hostVar, int deviceId,
                              hipDeviceptr_t* dev_ptr, size_t* size_ptr);

  bool getShadowVarInfo(std::string var_name, hipModule_t hmod,
                        void** var_addr, size_t* var_size);
  void setupArgument(const void *arg, size_t size, size_t offset);
  void configureCall(dim3 gridDim, dim3 blockDim, size_t sharedMem, hipStream_t stream);

  void popExec(ihipExec_t& exec);
};

/// Wait all active streams on the blocking queue. The method enqueues a wait command and
/// doesn't stall the current thread
extern void iHipWaitActiveStreams(amd::HostQueue* blocking_queue, bool wait_null_stream = false);

extern std::vector<hip::Device*> g_devices;
extern hipError_t ihipDeviceGetCount(int* count);
extern int ihipGetDevice();
extern hipError_t ihipMalloc(void** ptr, size_t sizeBytes, unsigned int flags);
extern amd::Memory* getMemoryObject(const void* ptr, size_t& offset);
extern bool CL_CALLBACK getSvarInfo(cl_program program, std::string var_name, void** var_addr,
                                    size_t* var_size);

#endif // HIP_SRC_HIP_INTERNAL_H
<commit_msg>SWDEV-216213 - Delete kernel function_ in hip::Function destructor<commit_after>/* Copyright (c) 2015-present Advanced Micro Devices, Inc.

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
 in the Software without restriction, including without limitation the rights
 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:

 The above copyright notice and this permission notice shall be included in
 all copies or substantial portions of the Software.

 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE. */

#ifndef HIP_SRC_HIP_INTERNAL_H
#define HIP_SRC_HIP_INTERNAL_H

#include "vdi_common.hpp"
#include "hip_prof_api.h"
#include "trace_helper.h"
#include "utils/debug.hpp"
#include "hip_formatting.hpp"
#include <unordered_set>
#include <thread>
#include <stack>
#include <mutex>
#include <iterator>
#ifdef _WIN32
#include <process.h>
#else
#include <unistd.h>
#endif

/*! IHIP IPC MEMORY Structure */
#define IHIP_IPC_MEM_HANDLE_SIZE   32
#define IHIP_IPC_MEM_RESERVED_SIZE LP64_SWITCH(28,24)

typedef struct ihipIpcMemHandle_st {
  char ipc_handle[IHIP_IPC_MEM_HANDLE_SIZE];  ///< ipc memory handle on ROCr
  size_t psize;
  char reserved[IHIP_IPC_MEM_RESERVED_SIZE];
} ihipIpcMemHandle_t;

#ifdef _WIN32
  inline int getpid() { return _getpid(); }
#endif

#define HIP_INIT() \
  std::call_once(hip::g_ihipInitialized, hip::init);       \
  if (hip::g_device == nullptr && g_devices.size() > 0) {  \
    hip::g_device = g_devices[0];                          \
  }

// This macro should be called at the beginning of every HIP API.
#define HIP_INIT_API(cid, ...)                               \
  ClPrint(amd::LOG_INFO, amd::LOG_API, "%-5d: [%zx] %s ( %s )", getpid(), std::this_thread::get_id(), __func__, ToString( __VA_ARGS__ ).c_str()); \
  amd::Thread* thread = amd::Thread::current();              \
  if (!VDI_CHECK_THREAD(thread)) {                           \
    HIP_RETURN(hipErrorOutOfMemory);                         \
  }                                                          \
  HIP_INIT()                                                 \
  HIP_CB_SPAWNER_OBJECT(cid);

#define HIP_RETURN(ret)          \
  hip::g_lastError = ret;  \
  ClPrint(amd::LOG_INFO, amd::LOG_API, "%-5d: [%zx] %s: Returned %s", getpid(), std::this_thread::get_id(), __func__, hipGetErrorName(hip::g_lastError)); \
  return hip::g_lastError;

namespace hc {
class accelerator;
class accelerator_view;
};

namespace hip {
  class Device;

  class Stream {
    amd::HostQueue* queue_;
    mutable amd::Monitor lock_;
    Device* device_;
    amd::CommandQueue::Priority priority_;
    unsigned int flags_;
    bool null_;

  public:
    Stream(Device* dev, amd::CommandQueue::Priority p, unsigned int f = 0, bool null_stream = false);
    bool Create();
    amd::HostQueue* asHostQueue();
    void Destroy();
    void Finish() const;
    /// Get device ID associated with the current stream;
    int DeviceId() const;
    /// Returns if stream is null stream
    bool Null() const { return null_; }
    /// Returns the lock object for the current stream
    amd::Monitor& Lock() const { return lock_; }
    /// Returns the creation flags for the current stream
    unsigned int Flags() const { return flags_; }
  };

  /// HIP Device class
  class Device {
    amd::Monitor lock_{"Device lock"};
    /// VDI context
    amd::Context* context_;
    /// Device's ID
    /// Store it here so we don't have to loop through the device list every time
    int deviceId_;
    /// VDI host queue for default streams
    Stream null_stream_;
    //Maintain list of user enabled peers
    std::list<int> userEnabledPeers;

  public:
    Device(amd::Context* ctx, int devId):
      context_(ctx), deviceId_(devId), null_stream_(this, amd::CommandQueue::Priority::Normal, 0, true)
        { assert(ctx != nullptr); }
    ~Device() {}

    amd::Context* asContext() const { return context_; }
    int deviceId() const { return deviceId_; }
    void retain() const { context_->retain(); }
    void release() const { context_->release(); }
    const std::vector<amd::Device*>& devices() const { return context_->devices(); }
    hipError_t EnablePeerAccess(int peerDeviceId){
      amd::ScopedLock lock(lock_);
      bool found = (std::find(userEnabledPeers.begin(), userEnabledPeers.end(), peerDeviceId) != userEnabledPeers.end());
      if (found) {
        return hipErrorPeerAccessAlreadyEnabled;
      }
      userEnabledPeers.push_back(peerDeviceId);
      return hipSuccess;
    }
    hipError_t DisablePeerAccess(int peerDeviceId) {
      amd::ScopedLock lock(lock_);
      bool found = (std::find(userEnabledPeers.begin(), userEnabledPeers.end(), peerDeviceId) != userEnabledPeers.end());
      if (found) {
        userEnabledPeers.remove(peerDeviceId);
        return hipSuccess;
      } else {
        return hipErrorPeerAccessNotEnabled;
      }
    }
    amd::HostQueue* NullStream();
  };

  extern std::once_flag g_ihipInitialized;
  /// Current thread's device
  extern thread_local Device* g_device;
  extern thread_local hipError_t g_lastError;
  /// Device representing the host - for pinned memory
  extern Device* host_device;

  extern void init();

  extern Device* getCurrentDevice();

  extern void setCurrentDevice(unsigned int index);

  /// Get VDI queue associated with hipStream
  /// Note: This follows the CUDA spec to sync with default streams
  ///       and Blocking streams
  extern amd::HostQueue* getQueue(hipStream_t s);
  /// Get default stream associated with the VDI context
  extern amd::HostQueue* getNullStream(amd::Context&);
  /// Get default stream of the thread
  extern amd::HostQueue* getNullStream();

  struct Function {
    amd::Kernel* function_;
    amd::Monitor lock_;

    Function(amd::Kernel* f) : function_(f), lock_("function lock") {}
    ~Function() { function_->release(); }
    hipFunction_t asHipFunction() { return reinterpret_cast<hipFunction_t>(this); }

    static Function* asFunction(hipFunction_t f) { return reinterpret_cast<Function*>(f); }
  };

};

struct ihipExec_t {
  dim3 gridDim_;
  dim3 blockDim_;
  size_t sharedMem_;
  hipStream_t hStream_;
  std::vector<char> arguments_;
};

class PlatformState {
  amd::Monitor lock_{"Guards global function map", true};

  std::unordered_map<const void*, std::vector<std::pair<hipModule_t, bool>>> modules_;
  bool initialized_{false};

  void digestFatBinary(const void* data, std::vector<std::pair<hipModule_t, bool>>& programs);
public:
  void init();
  std::vector<std::pair<hipModule_t, bool>>* addFatBinary(const void*data)
  {
    if (initialized_) {
      digestFatBinary(data, modules_[data]);
    }
    return &modules_[data];
  }
  void removeFatBinary(std::vector<std::pair<hipModule_t, bool>>* module)
  {
    for (auto& mod : modules_) {
      if (&mod.second == module) {
        modules_.erase(&mod);
        return;
      }
    }
  }

  struct RegisteredVar {
  public:
    RegisteredVar(): size_(0), devicePtr_(nullptr), amd_mem_obj_(nullptr) {}
    ~RegisteredVar() {}

    hipDeviceptr_t getdeviceptr() const { return devicePtr_; };
    size_t getvarsize() const { return size_; };

    size_t size_;               // Size of the variable
    hipDeviceptr_t devicePtr_;  //Device Memory Address of the variable.
    amd::Memory* amd_mem_obj_;
  };

  struct DeviceFunction {
    std::string deviceName;
    std::vector< std::pair< hipModule_t, bool > >* modules;
    std::vector<hipFunction_t> functions;
  };
  enum DeviceVarKind {
    DVK_Variable,
    DVK_Surface,
    DVK_Texture
  };
  struct DeviceVar {
    DeviceVarKind kind;
    void* shadowVptr;
    std::string hostVar;
    size_t size;
    std::vector< std::pair< hipModule_t, bool > >* modules;
    std::vector<RegisteredVar> rvars;
    bool dyn_undef;
    int type; // surface/texture type
    int norm; // texture has normalized output
    bool shadowAllocated = false; // shadow ptr is allocated on-demand and needs freeing.
  };
private:
  class Module {
  public:
    Module(hipModule_t hip_module_) : hip_module(hip_module_) {}
    std::unordered_map<std::string, DeviceFunction > functions_;
  private:
    hipModule_t hip_module;
  };
  std::unordered_map<hipModule_t, Module*> module_map_;

  std::unordered_map<const void*, DeviceFunction > functions_;
  std::unordered_multimap<std::string, DeviceVar > vars_;
  // Map from the host shadow symbol to its device name. As different modules
  // may have the same name, each symbol is uniquely identified by a pair of
  // module handle and its name.
  std::unordered_map<const void*,
                     std::pair<hipModule_t, std::string>> symbols_;

  static PlatformState* platform_;

  PlatformState() {}
  ~PlatformState() {}
public:
  static PlatformState& instance() {
    if (platform_ == nullptr) {
       // __hipRegisterFatBinary() will call this when app starts, thus
       // there is no multiple entry issue here.
       platform_ =  new PlatformState();
    }
    return *platform_;
  }

  bool unregisterFunc(hipModule_t hmod);
  std::vector< std::pair<hipModule_t, bool> >* unregisterVar(hipModule_t hmod);

  bool findSymbol(const void *hostVar, hipModule_t &hmod, std::string &devName);
  PlatformState::DeviceVar* findVar(std::string hostVar, int deviceId, hipModule_t hmod);
  void registerVarSym(const void *hostVar, hipModule_t hmod, const char *symbolName);
  void registerVar(const char* symbolName, const DeviceVar& var);
  void registerFunction(const void* hostFunction, const DeviceFunction& func);

  bool registerModFuncs(std::vector<std::string>& func_names, hipModule_t* module);
  bool findModFunc(hipFunction_t* hfunc, hipModule_t hmod, const char* name);
  bool createFunc(hipFunction_t* hfunc, hipModule_t hmod, const char* name);
  hipFunction_t getFunc(const void* hostFunction, int deviceId);
  bool getFuncAttr(const void* hostFunction, hipFuncAttributes* func_attr);
  bool getGlobalVar(const char* hostVar, int deviceId, hipModule_t hmod,
                    hipDeviceptr_t* dev_ptr, size_t* size_ptr);
  bool getTexRef(const char* hostVar, hipModule_t hmod, textureReference** texRef);

  bool getGlobalVarFromSymbol(const void* hostVar, int deviceId,
                              hipDeviceptr_t* dev_ptr, size_t* size_ptr);

  bool getShadowVarInfo(std::string var_name, hipModule_t hmod,
                        void** var_addr, size_t* var_size);
  void setupArgument(const void *arg, size_t size, size_t offset);
  void configureCall(dim3 gridDim, dim3 blockDim, size_t sharedMem, hipStream_t stream);

  void popExec(ihipExec_t& exec);
};

/// Wait all active streams on the blocking queue. The method enqueues a wait command and
/// doesn't stall the current thread
extern void iHipWaitActiveStreams(amd::HostQueue* blocking_queue, bool wait_null_stream = false);

extern std::vector<hip::Device*> g_devices;
extern hipError_t ihipDeviceGetCount(int* count);
extern int ihipGetDevice();
extern hipError_t ihipMalloc(void** ptr, size_t sizeBytes, unsigned int flags);
extern amd::Memory* getMemoryObject(const void* ptr, size_t& offset);
extern bool CL_CALLBACK getSvarInfo(cl_program program, std::string var_name, void** var_addr,
                                    size_t* var_size);

#endif // HIP_SRC_HIP_INTERNAL_H
<|endoftext|>


<commit_before>// Copyright (c) 2017 offa
// Copyright 2011 Ciaran McHale.
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions.
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

#include "danek/internal/UidIdentifierProcessor.h"
#include "danek/internal/Util.h"
#include "danek/internal/Compat.h"
#include "danek/ConfigurationException.h"
#include "danek/StringVector.h"
#include <sstream>
#include <iomanip>
#include <string.h>
#include <ctype.h>
#include <stdio.h>

namespace danek
{
    namespace
    {
        std::string formatCount(std::size_t count)
        {
            constexpr std::size_t numDigits = 9;
            std::ostringstream ss;
            ss << std::setw(numDigits) << std::setfill('0') << count;
            return ss.str();
        }
    }

    UidIdentifierProcessor::UidIdentifierProcessor() : m_count(0), m_uidToken("uid-")
    {
    }

    //----------------------------------------------------------------------
    // Spelling must be in one of the following forms:
    //      - "foo"                 -->  "foo"
    //      - "uid-<foo>"           -->  "uid-<digits>-<foo>"
    //      - "uid-<digits>-<foo>"  -->  "uid-<new-digits>-<foo>"
    // where "<foo>" does NOT start with a digit or "-"
    //----------------------------------------------------------------------

    void UidIdentifierProcessor::expand(StringBuffer& spelling)
    {
        //--------
        // Common case optimizations
        //--------
        if (strchr(spelling.str().c_str(), '.') == nullptr)
        {
            spelling = expandOne(spelling.str());
            return;
        }
        if (strstr(spelling.str().c_str(), "uid-") == nullptr)
        {
            return;
        }

        //--------
        // Let's break apart the scoped name, expand each local part
        // and then recombine the parts into an expanded scoped name.
        //--------
        StringBuffer buf;
        StringBuffer msg;
        StringBuffer result;

        StringVector vec{util::splitScopes(spelling.str())};
        const int len = vec.size();
        for (int i = 0; i < len; ++i)
        {
            buf = vec[i];
            try
            {
                buf = expandOne(buf.str());
            }
            catch (const ConfigurationException&)
            {
                msg << "'" << spelling << "' is not a legal identifier";
                throw ConfigurationException(msg.str());
            }
            result.append(buf);
            if (i < len - 1)
            {
                result.append(".");
            }
        }
        spelling = result;
    }

    std::string UidIdentifierProcessor::expandOne(const std::string& spelling)
    {
        //--------
        // If spelling does not start with "uid-" then do nothing.
        //--------
        if( std::equal(m_uidToken.cbegin(), m_uidToken.cend(), spelling.cbegin()) == false )
        {
            return spelling;
        }

        StringBuffer msg;
        msg << "'" << spelling << "' is not a legal identifier";

        //--------
        // Check for "uid-" (with no suffix), because that is illegal
        //--------
        if( spelling == m_uidToken )
        {
            throw ConfigurationException(msg.str());
        }
        if( spelling.at(m_uidToken.size()) == '-' )
        {
            // illegal: "uid--foo"
            throw ConfigurationException(msg.str());
        }

        if (!isdigit(spelling[4]))
        {
            //--------
            // "uid-foo"  --> "uid-<digits>-foo"
            //--------
            compat::checkAssertion(m_count < 1000 * 1000 * 1000);
            const std::string suffix = &(spelling.c_str()[4]); // deep copy

            std::stringstream ss;
            ss << "uid-" << formatCount(m_count) << "-" << suffix;
            ++m_count;
            return ss.str();
        }

        auto itr = std::next(spelling.cbegin(), 4); // skip over "uid-"
        std::size_t count = 0;

        while (isdigit(*itr))
        {
            std::advance(itr, 1);
            ++count;
        }
        compat::checkAssertion(count > 0);
        if (*itr == '\0' || *itr != '-')
        {
            // illegal: "uid-<digits>" or "uid-<digits>foo"
            throw ConfigurationException(msg.str());
        }
        std::advance(itr, 1); // point to just after "uid-<digits>-"
        if (*itr == '\0')
        {
            // illegal: "uid-<digits>-"
            throw ConfigurationException(msg.str());
        }
        if (*itr == '-')
        {
            // illegal: "uid-<digits>--"
            throw ConfigurationException(msg.str());
        }
        if (isdigit(*(itr)))
        {
            // illegal: "uid-<digits>-<digits>foo"
            throw ConfigurationException(msg.str());
        }

        compat::checkAssertion(m_count < 1000 * 1000 * 1000);
        const std::string suffix(itr, spelling.cend()); // deep copy just after "uid-<digits>-"
        const std::string result(m_uidToken + formatCount(m_count) + "-" + suffix);
        ++m_count;
        return result;
    }

    const char* UidIdentifierProcessor::unexpand(const char* spelling, StringBuffer& buf) const
    {
        //--------
        // Common case optimizations
        //--------
        if (strchr(spelling, '.') == nullptr)
        {
            return unexpandOne(spelling, buf);
        }
        if (strstr(spelling, "uid-") == nullptr)
        {
            return spelling;
        }

        //--------
        // Let's break apart the scoped name, unexpand each local part
        // and then recombine the parts into an unexpanded scoped name.
        //--------
        StringBuffer result;
        StringBuffer msg;
        const char* str;

        StringVector vec{util::splitScopes(spelling)};
        const int len = vec.size();
        for (int i = 0; i < len; ++i)
        {
            try
            {
                str = unexpandOne(vec[i].c_str(), buf);
            }
            catch (const ConfigurationException&)
            {
                msg << "'" << spelling << "' is not a legal identifier";
                throw ConfigurationException(msg.str());
            }
            result.append(str);
            if (i < len - 1)
            {
                result.append(".");
            }
        }
        buf = result.str();
        return buf.str().c_str();
    }

    const char* UidIdentifierProcessor::unexpandOne(const char* spelling, StringBuffer& buf) const

    {
        int count;
        const char* ptr;

        //--------
        // If spelling does not start with "uid-<digits>-" then do nothing.
        //--------
        ptr = spelling;
        if (strncmp(ptr, "uid-", 4) != 0)
        {
            return spelling;
        }
        ptr += 4; // skip over "uid-"
        count = 0;
        while (isdigit(*ptr))
        {
            ++ptr;
            ++count;
        }
        if (count == 0 || *ptr != '-')
        {
            return spelling;
        }

        //--------
        // Okay, let's returned a modified spelling.
        //--------
        StringBuffer suffix;

        suffix = (ptr + 1); // deep copy from just after "uid-<digits>-"
        buf.clear();
        buf << "uid-" << suffix;
        return buf.str().c_str();
    }
}
<commit_msg>Some more improvements.<commit_after>// Copyright (c) 2017 offa
// Copyright 2011 Ciaran McHale.
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions.
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

#include "danek/internal/UidIdentifierProcessor.h"
#include "danek/internal/Util.h"
#include "danek/internal/Compat.h"
#include "danek/ConfigurationException.h"
#include "danek/StringVector.h"
#include <sstream>
#include <iomanip>
#include <string.h>
#include <ctype.h>
#include <stdio.h>

namespace danek
{
    namespace
    {
        std::string formatCount(std::size_t count)
        {
            constexpr std::size_t numDigits = 9;
            std::ostringstream ss;
            ss << std::setw(numDigits) << std::setfill('0') << count;
            return ss.str();
        }
    }

    UidIdentifierProcessor::UidIdentifierProcessor() : m_count(0), m_uidToken("uid-")
    {
    }

    //----------------------------------------------------------------------
    // Spelling must be in one of the following forms:
    //      - "foo"                 -->  "foo"
    //      - "uid-<foo>"           -->  "uid-<digits>-<foo>"
    //      - "uid-<digits>-<foo>"  -->  "uid-<new-digits>-<foo>"
    // where "<foo>" does NOT start with a digit or "-"
    //----------------------------------------------------------------------

    void UidIdentifierProcessor::expand(StringBuffer& spelling)
    {
        //--------
        // Common case optimizations
        //--------
        if (strchr(spelling.str().c_str(), '.') == nullptr)
        {
            spelling = expandOne(spelling.str());
            return;
        }
        if (strstr(spelling.str().c_str(), "uid-") == nullptr)
        {
            return;
        }

        //--------
        // Let's break apart the scoped name, expand each local part
        // and then recombine the parts into an expanded scoped name.
        //--------
        StringBuffer buf;
        StringBuffer msg;
        StringBuffer result;

        StringVector vec{util::splitScopes(spelling.str())};
        const int len = vec.size();
        for (int i = 0; i < len; ++i)
        {
            buf = vec[i];
            try
            {
                buf = expandOne(buf.str());
            }
            catch (const ConfigurationException&)
            {
                msg << "'" << spelling << "' is not a legal identifier";
                throw ConfigurationException(msg.str());
            }
            result.append(buf);
            if (i < len - 1)
            {
                result.append(".");
            }
        }
        spelling = result;
    }

    std::string UidIdentifierProcessor::expandOne(const std::string& spelling)
    {
        // If spelling does not start with "uid-" then do nothing.
        if( std::equal(m_uidToken.cbegin(), m_uidToken.cend(), spelling.cbegin()) == false )
        {
            return spelling;
        }

        StringBuffer msg;
        msg << "'" << spelling << "' is not a legal identifier";

        // Check for "uid-" (with no suffix) and "uid--foo", because that is illegal
        if( spelling == m_uidToken || spelling.at(m_uidToken.size()) == '-' )
        {
            throw ConfigurationException(msg.str());
        }

        if (!isdigit(spelling[4]))
        {
            //--------
            // "uid-foo"  --> "uid-<digits>-foo"
            //--------
            compat::checkAssertion(m_count < 1000 * 1000 * 1000);
            const std::string suffix = &(spelling.c_str()[4]); // deep copy

            std::stringstream ss;
            ss << "uid-" << formatCount(m_count) << "-" << suffix;
            ++m_count;
            return ss.str();
        }

        auto itr = std::next(spelling.cbegin(), 4); // skip over "uid-"
        std::size_t count = 0;

        while (isdigit(*itr))
        {
            std::advance(itr, 1);
            ++count;
        }
        compat::checkAssertion(count > 0);
        if (*itr == '\0' || *itr != '-')
        {
            // illegal: "uid-<digits>" or "uid-<digits>foo"
            throw ConfigurationException(msg.str());
        }
        std::advance(itr, 1); // point to just after "uid-<digits>-"
        if (*itr == '\0')
        {
            // illegal: "uid-<digits>-"
            throw ConfigurationException(msg.str());
        }
        if (*itr == '-')
        {
            // illegal: "uid-<digits>--"
            throw ConfigurationException(msg.str());
        }
        if (isdigit(*(itr)))
        {
            // illegal: "uid-<digits>-<digits>foo"
            throw ConfigurationException(msg.str());
        }

        compat::checkAssertion(m_count < 1000 * 1000 * 1000);
        const std::string suffix(itr, spelling.cend()); // deep copy just after "uid-<digits>-"
        const std::string result(m_uidToken + formatCount(m_count) + "-" + suffix);
        ++m_count;
        return result;
    }

    const char* UidIdentifierProcessor::unexpand(const char* spelling, StringBuffer& buf) const
    {
        //--------
        // Common case optimizations
        //--------
        if (strchr(spelling, '.') == nullptr)
        {
            return unexpandOne(spelling, buf);
        }
        if (strstr(spelling, "uid-") == nullptr)
        {
            return spelling;
        }

        //--------
        // Let's break apart the scoped name, unexpand each local part
        // and then recombine the parts into an unexpanded scoped name.
        //--------
        StringBuffer result;
        StringBuffer msg;
        const char* str;

        StringVector vec{util::splitScopes(spelling)};
        const int len = vec.size();
        for (int i = 0; i < len; ++i)
        {
            try
            {
                str = unexpandOne(vec[i].c_str(), buf);
            }
            catch (const ConfigurationException&)
            {
                msg << "'" << spelling << "' is not a legal identifier";
                throw ConfigurationException(msg.str());
            }
            result.append(str);
            if (i < len - 1)
            {
                result.append(".");
            }
        }
        buf = result.str();
        return buf.str().c_str();
    }

    const char* UidIdentifierProcessor::unexpandOne(const char* spelling, StringBuffer& buf) const

    {
        int count;
        const char* ptr;

        //--------
        // If spelling does not start with "uid-<digits>-" then do nothing.
        //--------
        ptr = spelling;
        if (strncmp(ptr, "uid-", 4) != 0)
        {
            return spelling;
        }
        ptr += 4; // skip over "uid-"
        count = 0;
        while (isdigit(*ptr))
        {
            ++ptr;
            ++count;
        }
        if (count == 0 || *ptr != '-')
        {
            return spelling;
        }

        //--------
        // Okay, let's returned a modified spelling.
        //--------
        StringBuffer suffix;

        suffix = (ptr + 1); // deep copy from just after "uid-<digits>-"
        buf.clear();
        buf << "uid-" << suffix;
        return buf.str().c_str();
    }
}
<|endoftext|>