<commit_before>#define TP_QT4_ENABLE_LOWLEVEL_API

#include <TelepathyQt4/ChannelFactory>
#include <TelepathyQt4/Connection>
#include <TelepathyQt4/ConnectionLowlevel>
#include <TelepathyQt4/Contact>
#include <TelepathyQt4/ContactCapabilities>
#include <TelepathyQt4/ContactFactory>
#include <TelepathyQt4/ContactManager>
#include <TelepathyQt4/PendingContacts>
#include <TelepathyQt4/PendingReady>

#include <telepathy-glib/telepathy-glib.h>

#include <tests/lib/glib/contacts-conn.h>
#include <tests/lib/test.h>

using namespace Tp;

class TestContactsCapabilities : public Test
{
    Q_OBJECT

public:
    TestContactsCapabilities(QObject *parent = 0)
        : Test(parent), mConnService(0)
    { }

protected Q_SLOTS:
    void expectConnInvalidated();
    void expectPendingContactsFinished(Tp::PendingOperation *);

private Q_SLOTS:
    void initTestCase();
    void init();

    void testCapabilities();

    void cleanup();
    void cleanupTestCase();

private:
    QString mConnName, mConnPath;
    TpTestsContactsConnection *mConnService;
    ConnectionPtr mConn;
    QList<ContactPtr> mContacts;
};

void TestContactsCapabilities::expectConnInvalidated()
{
    mLoop->exit(0);
}

void TestContactsCapabilities::expectPendingContactsFinished(PendingOperation *op)
{
    if (!op->isFinished()) {
        qWarning() << "unfinished";
        mLoop->exit(1);
        return;
    }

    if (op->isError()) {
        qWarning().nospace() << op->errorName()
            << ": " << op->errorMessage();
        mLoop->exit(2);
        return;
    }

    if (!op->isValid()) {
        qWarning() << "inconsistent results";
        mLoop->exit(3);
        return;
    }

    qDebug() << "finished";
    PendingContacts *pending = qobject_cast<PendingContacts *>(op);
    mContacts = pending->contacts();

    mLoop->exit(0);
}

void TestContactsCapabilities::initTestCase()
{
    initTestCaseImpl();

    g_type_init();
    g_set_prgname("contacts-capabilities");
    tp_debug_set_flags("all");
    dbus_g_bus_get(DBUS_BUS_STARTER, 0);

    gchar *name;
    gchar *connPath;
    GError *error = 0;

    mConnService = TP_TESTS_CONTACTS_CONNECTION(g_object_new(
            TP_TESTS_TYPE_CONTACTS_CONNECTION,
            "account", "me@example.com",
            "protocol", "foo",
            NULL));
    QVERIFY(mConnService != 0);
    QVERIFY(tp_base_connection_register(TP_BASE_CONNECTION(mConnService),
                "foo", &name, &connPath, &error));
    QVERIFY(error == 0);

    QVERIFY(name != 0);
    QVERIFY(connPath != 0);

    mConnName = QLatin1String(name);
    mConnPath = QLatin1String(connPath);

    g_free(name);
    g_free(connPath);

    mConn = Connection::create(mConnName, mConnPath,
            ChannelFactory::create(QDBusConnection::sessionBus()),
            ContactFactory::create());
    QCOMPARE(mConn->isReady(), false);

    QVERIFY(connect(mConn->lowlevel()->requestConnect(),
                    SIGNAL(finished(Tp::PendingOperation*)),
                    SLOT(expectSuccessfulCall(Tp::PendingOperation*))));
    QCOMPARE(mLoop->exec(), 0);
    QCOMPARE(mConn->isReady(), true);

    QCOMPARE(mConn->status(), ConnectionStatusConnected);

    QVERIFY(mConn->contactManager()->supportedFeatures().contains(Contact::FeatureCapabilities));
}

void TestContactsCapabilities::init()
{
    initImpl();
}

static void freeRccList(GPtrArray *rccs)
{
    g_boxed_free(TP_ARRAY_TYPE_REQUESTABLE_CHANNEL_CLASS_LIST, rccs);
}

static void addTextChatClass(GPtrArray *classes, TpHandleType handle_type)
{
    GHashTable *fixed = tp_asv_new(
        TP_PROP_CHANNEL_CHANNEL_TYPE, G_TYPE_STRING, TP_IFACE_CHANNEL_TYPE_TEXT,
        TP_PROP_CHANNEL_TARGET_HANDLE_TYPE, G_TYPE_UINT, handle_type,
        NULL);

    const gchar * const allowed[] = { NULL };
    GValueArray *arr = tp_value_array_build(2,
        TP_HASH_TYPE_STRING_VARIANT_MAP, fixed,
        G_TYPE_STRV, allowed,
        G_TYPE_INVALID);

    g_hash_table_unref(fixed);

    g_ptr_array_add(classes, arr);
}

static GHashTable *createContactCapabilities(TpHandle *handles)
{
    GHashTable *capabilities = g_hash_table_new_full(NULL, NULL, NULL,
        (GDestroyNotify) freeRccList);

    /* Support private text chats */
    GPtrArray *caps1 = g_ptr_array_sized_new(2);
    addTextChatClass(caps1, TP_HANDLE_TYPE_CONTACT);
    g_hash_table_insert(capabilities, GUINT_TO_POINTER(handles[0]), caps1);

    /* Support text chatrooms */
    GPtrArray *caps2 = g_ptr_array_sized_new(1);
    g_hash_table_insert(capabilities, GUINT_TO_POINTER(handles[1]), caps2);

    /* Don't support anything */
    GPtrArray *caps3 = g_ptr_array_sized_new(0);
    g_hash_table_insert(capabilities, GUINT_TO_POINTER(handles[2]), caps3);

    return capabilities;
}

void TestContactsCapabilities::testCapabilities()
{
    QStringList ids = QStringList() << QLatin1String("alice")
        << QLatin1String("bob") << QLatin1String("chris");

    gboolean supportTextChat[] = { TRUE, FALSE, FALSE };

    TpHandleRepoIface *serviceRepo =
        tp_base_connection_get_handles(TP_BASE_CONNECTION(mConnService),
                TP_HANDLE_TYPE_CONTACT);
    TpHandle handles[] = { 0, 0, 0 };
    for (int i = 0; i < 3; i++) {
        handles[i] = tp_handle_ensure(serviceRepo, ids[i].toLatin1().constData(),
                NULL, NULL);
    }

    GHashTable *capabilities = createContactCapabilities(handles);
    tp_tests_contacts_connection_change_capabilities(mConnService, capabilities);

    PendingContacts *pending = mConn->contactManager()->contactsForIdentifiers(
            ids, Features() << Contact::FeatureCapabilities);
    QVERIFY(connect(pending,
                    SIGNAL(finished(Tp::PendingOperation*)),
                    SLOT(expectPendingContactsFinished(Tp::PendingOperation*))));
    QCOMPARE(mLoop->exec(), 0);

    for (int i = 0; i < mContacts.size(); i++) {
        ContactPtr contact = mContacts[i];

        QCOMPARE(contact->requestedFeatures().contains(Contact::FeatureCapabilities), true);
        QCOMPARE(contact->actualFeatures().contains(Contact::FeatureCapabilities), true);

        QCOMPARE(contact->capabilities().textChats(), supportTextChat[i]);
        QCOMPARE(contact->capabilities().streamedMediaCalls(), false);
        QCOMPARE(contact->capabilities().streamedMediaAudioCalls(), false);
        QCOMPARE(contact->capabilities().streamedMediaVideoCalls(), false);
        QCOMPARE(contact->capabilities().streamedMediaVideoCallsWithAudio(), false);
        QCOMPARE(contact->capabilities().upgradingStreamedMediaCalls(), false);
    }
}

void TestContactsCapabilities::cleanup()
{
    cleanupImpl();
}

void TestContactsCapabilities::cleanupTestCase()
{
    if (mConn) {
        // Disconnect and wait for invalidated
        QVERIFY(connect(mConn->lowlevel()->requestDisconnect(),
                        SIGNAL(finished(Tp::PendingOperation*)),
                        SLOT(expectSuccessfulCall(Tp::PendingOperation*))));
        QCOMPARE(mLoop->exec(), 0);

        if (mConn->isValid()) {
            QVERIFY(connect(mConn.data(),
                            SIGNAL(invalidated(Tp::DBusProxy *,
                                               const QString &, const QString &)),
                            SLOT(expectConnInvalidated())));
            QCOMPARE(mLoop->exec(), 0);
        }
    }

    if (mConnService != 0) {
        g_object_unref(mConnService);
        mConnService = 0;
    }

    cleanupTestCaseImpl();
}

QTEST_MAIN(TestContactsCapabilities)
#include "_gen/contacts-capabilities.cpp.moc.hpp"
<commit_msg>Don't leak caps hash table in TestContactsCapabilities<commit_after>#define TP_QT4_ENABLE_LOWLEVEL_API

#include <TelepathyQt4/ChannelFactory>
#include <TelepathyQt4/Connection>
#include <TelepathyQt4/ConnectionLowlevel>
#include <TelepathyQt4/Contact>
#include <TelepathyQt4/ContactCapabilities>
#include <TelepathyQt4/ContactFactory>
#include <TelepathyQt4/ContactManager>
#include <TelepathyQt4/PendingContacts>
#include <TelepathyQt4/PendingReady>

#include <telepathy-glib/telepathy-glib.h>

#include <tests/lib/glib/contacts-conn.h>
#include <tests/lib/test.h>

using namespace Tp;

class TestContactsCapabilities : public Test
{
    Q_OBJECT

public:
    TestContactsCapabilities(QObject *parent = 0)
        : Test(parent), mConnService(0)
    { }

protected Q_SLOTS:
    void expectConnInvalidated();
    void expectPendingContactsFinished(Tp::PendingOperation *);

private Q_SLOTS:
    void initTestCase();
    void init();

    void testCapabilities();

    void cleanup();
    void cleanupTestCase();

private:
    QString mConnName, mConnPath;
    TpTestsContactsConnection *mConnService;
    ConnectionPtr mConn;
    QList<ContactPtr> mContacts;
};

void TestContactsCapabilities::expectConnInvalidated()
{
    mLoop->exit(0);
}

void TestContactsCapabilities::expectPendingContactsFinished(PendingOperation *op)
{
    if (!op->isFinished()) {
        qWarning() << "unfinished";
        mLoop->exit(1);
        return;
    }

    if (op->isError()) {
        qWarning().nospace() << op->errorName()
            << ": " << op->errorMessage();
        mLoop->exit(2);
        return;
    }

    if (!op->isValid()) {
        qWarning() << "inconsistent results";
        mLoop->exit(3);
        return;
    }

    qDebug() << "finished";
    PendingContacts *pending = qobject_cast<PendingContacts *>(op);
    mContacts = pending->contacts();

    mLoop->exit(0);
}

void TestContactsCapabilities::initTestCase()
{
    initTestCaseImpl();

    g_type_init();
    g_set_prgname("contacts-capabilities");
    tp_debug_set_flags("all");
    dbus_g_bus_get(DBUS_BUS_STARTER, 0);

    gchar *name;
    gchar *connPath;
    GError *error = 0;

    mConnService = TP_TESTS_CONTACTS_CONNECTION(g_object_new(
            TP_TESTS_TYPE_CONTACTS_CONNECTION,
            "account", "me@example.com",
            "protocol", "foo",
            NULL));
    QVERIFY(mConnService != 0);
    QVERIFY(tp_base_connection_register(TP_BASE_CONNECTION(mConnService),
                "foo", &name, &connPath, &error));
    QVERIFY(error == 0);

    QVERIFY(name != 0);
    QVERIFY(connPath != 0);

    mConnName = QLatin1String(name);
    mConnPath = QLatin1String(connPath);

    g_free(name);
    g_free(connPath);

    mConn = Connection::create(mConnName, mConnPath,
            ChannelFactory::create(QDBusConnection::sessionBus()),
            ContactFactory::create());
    QCOMPARE(mConn->isReady(), false);

    QVERIFY(connect(mConn->lowlevel()->requestConnect(),
                    SIGNAL(finished(Tp::PendingOperation*)),
                    SLOT(expectSuccessfulCall(Tp::PendingOperation*))));
    QCOMPARE(mLoop->exec(), 0);
    QCOMPARE(mConn->isReady(), true);

    QCOMPARE(mConn->status(), ConnectionStatusConnected);

    QVERIFY(mConn->contactManager()->supportedFeatures().contains(Contact::FeatureCapabilities));
}

void TestContactsCapabilities::init()
{
    initImpl();
}

static void freeRccList(GPtrArray *rccs)
{
    g_boxed_free(TP_ARRAY_TYPE_REQUESTABLE_CHANNEL_CLASS_LIST, rccs);
}

static void addTextChatClass(GPtrArray *classes, TpHandleType handle_type)
{
    GHashTable *fixed = tp_asv_new(
        TP_PROP_CHANNEL_CHANNEL_TYPE, G_TYPE_STRING, TP_IFACE_CHANNEL_TYPE_TEXT,
        TP_PROP_CHANNEL_TARGET_HANDLE_TYPE, G_TYPE_UINT, handle_type,
        NULL);

    const gchar * const allowed[] = { NULL };
    GValueArray *arr = tp_value_array_build(2,
        TP_HASH_TYPE_STRING_VARIANT_MAP, fixed,
        G_TYPE_STRV, allowed,
        G_TYPE_INVALID);

    g_hash_table_unref(fixed);

    g_ptr_array_add(classes, arr);
}

static GHashTable *createContactCapabilities(TpHandle *handles)
{
    GHashTable *capabilities = g_hash_table_new_full(NULL, NULL, NULL,
        (GDestroyNotify) freeRccList);

    /* Support private text chats */
    GPtrArray *caps1 = g_ptr_array_sized_new(2);
    addTextChatClass(caps1, TP_HANDLE_TYPE_CONTACT);
    g_hash_table_insert(capabilities, GUINT_TO_POINTER(handles[0]), caps1);

    /* Support text chatrooms */
    GPtrArray *caps2 = g_ptr_array_sized_new(1);
    g_hash_table_insert(capabilities, GUINT_TO_POINTER(handles[1]), caps2);

    /* Don't support anything */
    GPtrArray *caps3 = g_ptr_array_sized_new(0);
    g_hash_table_insert(capabilities, GUINT_TO_POINTER(handles[2]), caps3);

    return capabilities;
}

void TestContactsCapabilities::testCapabilities()
{
    QStringList ids = QStringList() << QLatin1String("alice")
        << QLatin1String("bob") << QLatin1String("chris");

    gboolean supportTextChat[] = { TRUE, FALSE, FALSE };

    TpHandleRepoIface *serviceRepo =
        tp_base_connection_get_handles(TP_BASE_CONNECTION(mConnService),
                TP_HANDLE_TYPE_CONTACT);
    TpHandle handles[] = { 0, 0, 0 };
    for (int i = 0; i < 3; i++) {
        handles[i] = tp_handle_ensure(serviceRepo, ids[i].toLatin1().constData(),
                NULL, NULL);
    }

    GHashTable *capabilities = createContactCapabilities(handles);
    tp_tests_contacts_connection_change_capabilities(mConnService, capabilities);
    g_hash_table_destroy(capabilities);

    PendingContacts *pending = mConn->contactManager()->contactsForIdentifiers(
            ids, Features() << Contact::FeatureCapabilities);
    QVERIFY(connect(pending,
                    SIGNAL(finished(Tp::PendingOperation*)),
                    SLOT(expectPendingContactsFinished(Tp::PendingOperation*))));
    QCOMPARE(mLoop->exec(), 0);

    for (int i = 0; i < mContacts.size(); i++) {
        ContactPtr contact = mContacts[i];

        QCOMPARE(contact->requestedFeatures().contains(Contact::FeatureCapabilities), true);
        QCOMPARE(contact->actualFeatures().contains(Contact::FeatureCapabilities), true);

        QCOMPARE(contact->capabilities().textChats(), supportTextChat[i]);
        QCOMPARE(contact->capabilities().streamedMediaCalls(), false);
        QCOMPARE(contact->capabilities().streamedMediaAudioCalls(), false);
        QCOMPARE(contact->capabilities().streamedMediaVideoCalls(), false);
        QCOMPARE(contact->capabilities().streamedMediaVideoCallsWithAudio(), false);
        QCOMPARE(contact->capabilities().upgradingStreamedMediaCalls(), false);
    }
}

void TestContactsCapabilities::cleanup()
{
    cleanupImpl();
}

void TestContactsCapabilities::cleanupTestCase()
{
    if (mConn) {
        // Disconnect and wait for invalidated
        QVERIFY(connect(mConn->lowlevel()->requestDisconnect(),
                        SIGNAL(finished(Tp::PendingOperation*)),
                        SLOT(expectSuccessfulCall(Tp::PendingOperation*))));
        QCOMPARE(mLoop->exec(), 0);

        if (mConn->isValid()) {
            QVERIFY(connect(mConn.data(),
                            SIGNAL(invalidated(Tp::DBusProxy *,
                                               const QString &, const QString &)),
                            SLOT(expectConnInvalidated())));
            QCOMPARE(mLoop->exec(), 0);
        }
    }

    if (mConnService != 0) {
        g_object_unref(mConnService);
        mConnService = 0;
    }

    cleanupTestCaseImpl();
}

QTEST_MAIN(TestContactsCapabilities)
#include "_gen/contacts-capabilities.cpp.moc.hpp"
<|endoftext|>


<commit_before>/***********************************************************************************************************************
 * Copyright (C) 2016 Andrew Zonenberg and contributors                                                                *
 *                                                                                *
 * This program is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General   *
 * Public License as published by the Free Software Foundation; either version 2.1 of the License, or (at your option) *
 * any later version.                                                                                *
 *                                                                                *
 * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied  *
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for     *
 * more details.                                                                                *
 *                                                                                *
 * You should have received a copy of the GNU Lesser General Public License along with this program; if not, you may   *
 * find one here:                                                                                *
 * https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt                                                              *
 * or you may search the http://www.gnu.org website for the version 2.1 license, or you may write to the Free Software *
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA                                      *
 **********************************************************************************************************************/

#include <log.h>
#include <gpdevboard.h>
#include <unistd.h>
#include <math.h>

using namespace std;

bool RunTest(hdevice hdev);

int main(int argc, char* argv[])
{
	g_log_sinks.emplace(g_log_sinks.begin(), new STDLogSink(Severity::VERBOSE));

	//expect one arg: the bitstream
	if(argc != 2)
	{
		LogNotice("Usage: [testcase] bitstream.txt\n");
		return 1;
	}

	//Set up the test
	hdevice hdev = MultiBoardTestSetup(argv[1], 25000, 3.3, SilegoPart::SLG46620V);
	if(!hdev)
	{
		LogError("Failed to open board\n");
		return 1;
	}

	//Run the actual test case
	LogNotice("\n");
	LogNotice("Running application test case\n");
	if(!RunTest(hdev))
	{
		SetStatusLED(hdev, 0);
		Reset(hdev);
		return 1;
	}

	//Turn off the LED before declaring success
	LogVerbose("\n");
	LogVerbose("Test complete, resetting board\n");
	SetStatusLED(hdev, 0);
	ResetAllSiggens(hdev);
	Reset(hdev);
	USBCleanup(hdev);
	return 0;
}

/**
	@brief The actual application-layer test
 */
bool RunTest(hdevice hdev)
{
	LogIndenter li;

	//Set up the I/O config we need for this test
	IOConfig ioConfig;
	for(size_t i = 2; i <= 20; i++)
		ioConfig.driverConfigs[i] = TP_RESET;
	if(!SetIOConfig(hdev, ioConfig))
		return false;

	double tolerance_trip = 0.05;
	double tolerance_vref = 0.15;

	//Check the bandgap outputs
	LogVerbose("Testing bandgap status\n");
	double v;
	if(!SingleReadADC(hdev, 20, v))
		return false;
	if(v < 0.5)
	{
		LogError("Bandgap status should be high\n");
		return false;
	}

	//Check the Vref outputs
	//TODO: Why is the error so wide here?
	LogVerbose("Checking ref outputs\n");
	{
		LogIndenter li;

		if(!SingleReadADC(hdev, 19, v))
			return false;
		double err = 0.8 - v;
		LogVerbose("Vref_800 = %.3f V (err = %.3f)\n", v, err);
		if(fabs(err) > tolerance_vref)
		{
			LogError("Error is too big\n");
			return false;
		}

		if(!SingleReadADC(hdev, 18, v))
			return false;
		err = 0.6 - v;
		LogVerbose("Vref_600 = %.3f V (err = %.3f)\n", v, err);
		if(fabs(err) > tolerance_vref)
		{
			LogError("Error is too big\n");
			return false;
		}
	}

	//(pin num, threshold in mV)
	const int acmps[6][2] =
	{
		{12, 200},
		{13, 400},
		{14, 600},
		{15, 800},
		{16, 1000},
		{17, 1000}
	};

	//Sweep input and verify output
	LogVerbose("Running comparator voltage sweep\n");
	double step = 0.025;
	bool pin_states[5] = {false};
	for(double vtest = step; vtest < 1.1; vtest += step)
	{
		LogIndenter li;

		//Set up a signal generator on the inputs
		//TODO: check 6 and 4 independently?
		if(!ConfigureSiggen(hdev, 6, vtest))
			return false;
		if(!ConfigureSiggen(hdev, 4, vtest))
			return false;

		//Read the outputs
		for(int i=0; i<6; i++)
		{
			double v;
			int npin = acmps[i][0];
			if(!SingleReadADC(hdev, npin, v))
				return false;

			bool b = (v > 0.5);

			//If we went LOW, something is wrong. We should never turn off on a rising edge
			if(pin_states[i] && !b)
			{
				LogError("Pin %d went low at %.3f V on a rising input waveform - something is wrong\n",
					npin, vtest);
				return false;
			}

			//If we went HIGH, log the trip point and check if it's sane
			if(!pin_states[i] && b)
			{
				double expected_trip = acmps[i][1] * 0.001f;
				double delta = vtest - expected_trip;

				LogVerbose("Pin %d went high at %.3f V (err = %.3f V)\n", npin, vtest, delta);

				if(fabs(delta) > tolerance_trip)
				{
					LogError("Error is too big\n");
					return false;
				}
			}

			//Save pin state
			pin_states[i] = b;
		}

		/*
		//Read the input
		if(!SelectADCChannel(hdev, INPUT_PIN))
			return false;
		double vin;
		if(!ReadADC(hdev, vin))
			return false;

		//Read the output
		if(!SelectADCChannel(hdev, OUTPUT_PIN))
			return false;
		double vout;
		if(!ReadADC(hdev, vout))
			return false;

		double expected = vin * 2;
		double error = expected - vout;
		double pctError = (error * 100) / expected;

		LogVerbose("|%10.3f|%10.3f|%10.3f|%10.3f|%10.3f|%10.3f|\n", vtest, vin, vout, expected, error, pctError);

		//TODO: Decide on an appropriate tolerance
		if(pctError > 5)
		{
			LogError("Percent error is too big, test failed\n");
			return false;
		}*/
	}

	return true;
}
<commit_msg>tests: fixed out of bounds array access<commit_after>/***********************************************************************************************************************
 * Copyright (C) 2016 Andrew Zonenberg and contributors                                                                *
 *                                                                                *
 * This program is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General   *
 * Public License as published by the Free Software Foundation; either version 2.1 of the License, or (at your option) *
 * any later version.                                                                                *
 *                                                                                *
 * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied  *
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for     *
 * more details.                                                                                *
 *                                                                                *
 * You should have received a copy of the GNU Lesser General Public License along with this program; if not, you may   *
 * find one here:                                                                                *
 * https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt                                                              *
 * or you may search the http://www.gnu.org website for the version 2.1 license, or you may write to the Free Software *
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA                                      *
 **********************************************************************************************************************/

#include <log.h>
#include <gpdevboard.h>
#include <unistd.h>
#include <math.h>

using namespace std;

bool RunTest(hdevice hdev);

int main(int argc, char* argv[])
{
	g_log_sinks.emplace(g_log_sinks.begin(), new STDLogSink(Severity::VERBOSE));

	//expect one arg: the bitstream
	if(argc != 2)
	{
		LogNotice("Usage: [testcase] bitstream.txt\n");
		return 1;
	}

	//Set up the test
	hdevice hdev = MultiBoardTestSetup(argv[1], 25000, 3.3, SilegoPart::SLG46620V);
	if(!hdev)
	{
		LogError("Failed to open board\n");
		return 1;
	}

	//Run the actual test case
	LogNotice("\n");
	LogNotice("Running application test case\n");
	if(!RunTest(hdev))
	{
		SetStatusLED(hdev, 0);
		Reset(hdev);
		return 1;
	}

	//Turn off the LED before declaring success
	LogVerbose("\n");
	LogVerbose("Test complete, resetting board\n");
	SetStatusLED(hdev, 0);
	ResetAllSiggens(hdev);
	Reset(hdev);
	USBCleanup(hdev);
	return 0;
}

/**
	@brief The actual application-layer test
 */
bool RunTest(hdevice hdev)
{
	LogIndenter li;

	//Set up the I/O config we need for this test
	IOConfig ioConfig;
	for(size_t i = 2; i <= 20; i++)
		ioConfig.driverConfigs[i] = TP_RESET;
	if(!SetIOConfig(hdev, ioConfig))
		return false;

	double tolerance_trip = 0.05;
	double tolerance_vref = 0.15;

	//Check the bandgap outputs
	LogVerbose("Testing bandgap status\n");
	double v;
	if(!SingleReadADC(hdev, 20, v))
		return false;
	if(v < 0.5)
	{
		LogError("Bandgap status should be high\n");
		return false;
	}

	//Check the Vref outputs
	//TODO: Why is the error so wide here?
	LogVerbose("Checking ref outputs\n");
	{
		LogIndenter li;

		if(!SingleReadADC(hdev, 19, v))
			return false;
		double err = 0.8 - v;
		LogVerbose("Vref_800 = %.3f V (err = %.3f)\n", v, err);
		if(fabs(err) > tolerance_vref)
		{
			LogError("Error is too big\n");
			return false;
		}

		if(!SingleReadADC(hdev, 18, v))
			return false;
		err = 0.6 - v;
		LogVerbose("Vref_600 = %.3f V (err = %.3f)\n", v, err);
		if(fabs(err) > tolerance_vref)
		{
			LogError("Error is too big\n");
			return false;
		}
	}

	//(pin num, threshold in mV)
	const int acmps[6][2] =
	{
		{12, 200},
		{13, 400},
		{14, 600},
		{15, 800},
		{16, 1000},
		{17, 1000}
	};

	//Sweep input and verify output
	LogVerbose("Running comparator voltage sweep\n");
	double step = 0.025;
	bool pin_states[6] = {false};
	for(double vtest = step; vtest < 1.1; vtest += step)
	{
		LogIndenter li;

		//Set up a signal generator on the inputs
		//TODO: check 6 and 4 independently?
		if(!ConfigureSiggen(hdev, 6, vtest))
			return false;
		if(!ConfigureSiggen(hdev, 4, vtest))
			return false;

		//Read the outputs
		for(int i=0; i<6; i++)
		{
			double v;
			int npin = acmps[i][0];
			if(!SingleReadADC(hdev, npin, v))
				return false;

			bool b = (v > 0.5);

			//If we went LOW, something is wrong. We should never turn off on a rising edge
			if(pin_states[i] && !b)
			{
				LogError("Pin %d went low at %.3f V on a rising input waveform - something is wrong\n",
					npin, vtest);
				return false;
			}

			//If we went HIGH, log the trip point and check if it's sane
			if(!pin_states[i] && b)
			{
				double expected_trip = acmps[i][1] * 0.001f;
				double delta = vtest - expected_trip;

				LogVerbose("Pin %d went high at %.3f V (err = %.3f V)\n", npin, vtest, delta);

				if(fabs(delta) > tolerance_trip)
				{
					LogError("Error is too big\n");
					return false;
				}
			}

			//Save pin state
			pin_states[i] = b;
		}

		/*
		//Read the input
		if(!SelectADCChannel(hdev, INPUT_PIN))
			return false;
		double vin;
		if(!ReadADC(hdev, vin))
			return false;

		//Read the output
		if(!SelectADCChannel(hdev, OUTPUT_PIN))
			return false;
		double vout;
		if(!ReadADC(hdev, vout))
			return false;

		double expected = vin * 2;
		double error = expected - vout;
		double pctError = (error * 100) / expected;

		LogVerbose("|%10.3f|%10.3f|%10.3f|%10.3f|%10.3f|%10.3f|\n", vtest, vin, vout, expected, error, pctError);

		//TODO: Decide on an appropriate tolerance
		if(pctError > 5)
		{
			LogError("Percent error is too big, test failed\n");
			return false;
		}*/
	}

	return true;
}
<|endoftext|>


<commit_before>//! \file
/*
**  Copyright (C) - Triton
**
**  This program is under the terms of the BSD License.
*/

#include <new>

#include <triton/exceptions.hpp>
#include <triton/irBuilder.hpp>
#include <triton/memoryAccess.hpp>
#include <triton/operandWrapper.hpp>
#include <triton/register.hpp>
#include <triton/x86Semantics.hpp>
#include <triton/astContext.hpp>

namespace triton {
  namespace arch {

    IrBuilder::IrBuilder(triton::arch::Architecture* architecture,
                         const triton::modes::Modes& modes,
                         triton::ast::AstContext& astCtxt,
                         triton::engines::symbolic::SymbolicEngine* symbolicEngine,
                         triton::engines::taint::TaintEngine* taintEngine)
      : modes(modes),
        astGarbageCollector(astCtxt.getAstGarbageCollector()),
        backupAstGarbageCollector(modes, true) {

      if (architecture == nullptr)
        throw triton::exceptions::IrBuilder("IrBuilder::IrBuilder(): The architecture API must be defined.");

      if (symbolicEngine == nullptr)
        throw triton::exceptions::IrBuilder("IrBuilder::IrBuilder(): The symbolic engine API must be defined.");

      if (taintEngine == nullptr)
        throw triton::exceptions::IrBuilder("IrBuilder::IrBuilder(): The taint engines API must be defined.");

      this->architecture              = architecture;
      this->backupSymbolicEngine      = new(std::nothrow) triton::engines::symbolic::SymbolicEngine(architecture, modes, astCtxt, nullptr, true);
      this->symbolicEngine            = symbolicEngine;
      this->taintEngine               = taintEngine;
      this->x86Isa                    = new(std::nothrow) triton::arch::x86::x86Semantics(architecture, symbolicEngine, taintEngine, astCtxt);

      if (this->x86Isa == nullptr || this->backupSymbolicEngine == nullptr)
        throw triton::exceptions::IrBuilder("IrBuilder::IrBuilder(): Not enough memory.");
    }

    IrBuilder::~IrBuilder() {
      delete this->backupSymbolicEngine;
      delete this->x86Isa;
    }

    bool IrBuilder::buildSemantics(triton::arch::Instruction& inst) {
      bool ret = false;

      if (this->architecture->getArchitecture() == triton::arch::ARCH_INVALID)
        throw triton::exceptions::IrBuilder("IrBuilder::buildSemantics(): You must define an architecture.");

      /* Stage 1 - Update the context memory */
      std::list<triton::arch::MemoryAccess>::iterator it1;
      for (it1 = inst.memoryAccess.begin(); it1 != inst.memoryAccess.end(); it1++) {
        this->architecture->setConcreteMemoryValue(*it1);
      }

      /* Stage 2 - Update the context register */
      std::map<triton::uint32, triton::arch::Register>::iterator it2;
      for (it2 = inst.registerState.begin(); it2 != inst.registerState.end(); it2++) {
        this->architecture->setConcreteRegisterValue(it2->second);
      }

      /* Stage 3 - Initialize the target address of memory operands */
      std::vector<triton::arch::OperandWrapper>::iterator it3;
      for (it3 = inst.operands.begin(); it3 != inst.operands.end(); it3++) {
        if (it3->getType() == triton::arch::OP_MEM) {
          this->symbolicEngine->initLeaAst(it3->getMemory());
        }
      }

      /* Pre IR processing */
      this->preIrInit(inst);

      /* Processing */
      switch (this->architecture->getArchitecture()) {
        case triton::arch::ARCH_X86:
        case triton::arch::ARCH_X86_64:
          ret = this->x86Isa->buildSemantics(inst);
      }

      /* Post IR processing */
      this->postIrInit(inst);

      return ret;
    }

    void IrBuilder::preIrInit(triton::arch::Instruction& inst) {
      /* Clear previous expressions if exist */
      inst.symbolicExpressions.clear();

      /* Clear implicit and explicit semantics */
      inst.getLoadAccess().clear();
      inst.getReadRegisters().clear();
      inst.getReadImmediates().clear();
      inst.getStoreAccess().clear();
      inst.getWrittenRegisters().clear();

      /* Update instruction address if undefined */
      if (!inst.getAddress())
        inst.setAddress(this->architecture->getConcreteRegisterValue(this->architecture->getParentRegister(ID_REG_IP)).convert_to<triton::uint64>());

      /* Backup the symbolic engine in the case where only the taint is available. */
      if (!this->symbolicEngine->isEnabled()) {
        *this->backupSymbolicEngine = *this->symbolicEngine;
        this->backupAstGarbageCollector = this->astGarbageCollector;
      }
    }

    void IrBuilder::postIrInit(triton::arch::Instruction& inst) {
      std::set<triton::ast::AbstractNode*> uniqueNodes;
      std::vector<triton::engines::symbolic::SymbolicExpression*> newVector;

      auto& loadAccess        = inst.getLoadAccess();
      auto& readRegisters     = inst.getReadRegisters();
      auto& readImmediates    = inst.getReadImmediates();
      auto& storeAccess       = inst.getStoreAccess();
      auto& writtenRegisters  = inst.getWrittenRegisters();

      /* Clear unused data */
      inst.memoryAccess.clear();
      inst.registerState.clear();

      /* Set the taint */
      inst.setTaint();

      // ----------------------------------------------------------------------

      /*
       * If the symbolic engine is disable we delete symbolic
       * expressions and AST nodes. Note that if the taint engine
       * is enable we must compute semanitcs to spread the taint.
       */
      if (!this->symbolicEngine->isEnabled()) {
        /* Clear memory operands */
        this->collectNodes(uniqueNodes, inst.operands, false);

        /* Clear implicit and explicit semantics */
        loadAccess.clear();
        readRegisters.clear();
        readImmediates.clear();
        storeAccess.clear();
        writtenRegisters.clear();

        /* Symbolic Expressions */
        this->removeSymbolicExpressions(inst, uniqueNodes);

        /* Restore backup */
        *this->symbolicEngine = *this->backupSymbolicEngine;
      }

      // ----------------------------------------------------------------------

      /*
       * If the symbolic engine is defined to process symbolic
       * execution only on symbolized expressions, we delete all
       * concrete expressions and their AST nodes.
       */
      if (this->symbolicEngine->isEnabled() && this->modes.isModeEnabled(triton::modes::ONLY_ON_SYMBOLIZED)) {
        /* Clear memory operands */
        this->collectUnsymbolizedNodes(uniqueNodes, inst.operands);

        /* Clear implicit and explicit semantics - MEM */
        this->collectUnsymbolizedNodes(loadAccess);

        /* Clear implicit and explicit semantics - REG */
        this->collectUnsymbolizedNodes(readRegisters);

        /* Clear implicit and explicit semantics - IMM */
        this->collectUnsymbolizedNodes(readImmediates);

        /* Clear implicit and explicit semantics - MEM */
        this->collectUnsymbolizedNodes(storeAccess);

        /* Clear implicit and explicit semantics - REG */
        this->collectUnsymbolizedNodes(writtenRegisters);

        /* Clear symbolic expressions */
        for (auto it = inst.symbolicExpressions.cbegin(); it != inst.symbolicExpressions.cend(); it++) {
          if ((*it)->isSymbolized() == false) {
            this->astGarbageCollector.extractUniqueAstNodes(uniqueNodes, (*it)->getAst());
            this->symbolicEngine->removeSymbolicExpression((*it)->getId());
          }
          else
            newVector.push_back(*it);
        }
        inst.symbolicExpressions = newVector;
      }

      // ----------------------------------------------------------------------

      /*
       * If the symbolic engine is defined to process symbolic
       * execution only on tainted instructions, we delete all
       * expressions untainted and their AST nodes.
       */
      else if (this->modes.isModeEnabled(triton::modes::ONLY_ON_TAINTED) && !inst.isTainted()) {
        /* Memory operands */
        this->collectNodes(uniqueNodes, inst.operands, true);

        /* Implicit and explicit semantics - MEM */
        this->collectNodes(uniqueNodes, loadAccess);

        /* Implicit and explicit semantics - REG */
        this->collectNodes(uniqueNodes, readRegisters);

        /* Implicit and explicit semantics - IMM */
        this->collectNodes(uniqueNodes, readImmediates);

        /* Implicit and explicit semantics - MEM */
        this->collectNodes(uniqueNodes, storeAccess);

        /* Implicit and explicit semantics - REG */
        this->collectNodes(uniqueNodes, writtenRegisters);

        /* Symbolic Expressions */
        this->removeSymbolicExpressions(inst, uniqueNodes);
      }

      // ----------------------------------------------------------------------

      /* Free collected nodes */
      this->astGarbageCollector.freeAstNodes(uniqueNodes);

      if (!this->symbolicEngine->isEnabled())
        this->astGarbageCollector = this->backupAstGarbageCollector;
    }

    void IrBuilder::removeSymbolicExpressions(triton::arch::Instruction& inst, std::set<triton::ast::AbstractNode*>& uniqueNodes) {
      for (auto it = inst.symbolicExpressions.cbegin(); it != inst.symbolicExpressions.cend(); it++) {
        this->astGarbageCollector.extractUniqueAstNodes(uniqueNodes, (*it)->getAst());
        this->symbolicEngine->removeSymbolicExpression((*it)->getId());
      }
      inst.symbolicExpressions.clear();
    }

    template <typename T>
    void IrBuilder::collectNodes(std::set<triton::ast::AbstractNode*>& uniqueNodes, T& items) const {
      for (auto it = items.cbegin(); it != items.cend(); it++)
        this->astGarbageCollector.extractUniqueAstNodes(uniqueNodes, std::get<1>(*it));
      items.clear();
    }

    void IrBuilder::collectNodes(std::set<triton::ast::AbstractNode*>& uniqueNodes, std::vector<triton::arch::OperandWrapper>& operands, bool gc) const {
      for (auto it = operands.begin(); it != operands.end(); it++) {
        if (it->getType() == triton::arch::OP_MEM) {
          if (gc)
            this->astGarbageCollector.extractUniqueAstNodes(uniqueNodes, it->getMemory().getLeaAst());
          it->getMemory().setLeaAst(nullptr);
        }
      }
    }

    template <typename T>
    void IrBuilder::collectUnsymbolizedNodes(T& items) const {
      T newItems;

      for (auto it = items.cbegin(); it != items.cend(); it++) {
        if (std::get<1>(*it) && std::get<1>(*it)->isSymbolized() == true)
          newItems.insert(*it);
      }

      items.clear();
      items = newItems;
    }

    void IrBuilder::collectUnsymbolizedNodes(std::set<triton::ast::AbstractNode*>& uniqueNodes, std::vector<triton::arch::OperandWrapper>& operands) const {
      for (auto it = operands.begin(); it!= operands.end(); it++) {
        if (it->getType() == triton::arch::OP_MEM) {
          if (it->getMemory().getLeaAst() && it->getMemory().getLeaAst()->isSymbolized() == false) {
            this->astGarbageCollector.extractUniqueAstNodes(uniqueNodes, it->getMemory().getLeaAst());
            it->getMemory().setLeaAst(nullptr);
          }
        }
      }
    }

  }; /* arch namespace */
}; /* triton namespace */
<commit_msg>Code Review: Use autofor<commit_after>//! \file
/*
**  Copyright (C) - Triton
**
**  This program is under the terms of the BSD License.
*/

#include <new>

#include <triton/exceptions.hpp>
#include <triton/irBuilder.hpp>
#include <triton/memoryAccess.hpp>
#include <triton/operandWrapper.hpp>
#include <triton/register.hpp>
#include <triton/x86Semantics.hpp>
#include <triton/astContext.hpp>

namespace triton {
  namespace arch {

    IrBuilder::IrBuilder(triton::arch::Architecture* architecture,
                         const triton::modes::Modes& modes,
                         triton::ast::AstContext& astCtxt,
                         triton::engines::symbolic::SymbolicEngine* symbolicEngine,
                         triton::engines::taint::TaintEngine* taintEngine)
      : modes(modes),
        astGarbageCollector(astCtxt.getAstGarbageCollector()),
        backupAstGarbageCollector(modes, true) {

      if (architecture == nullptr)
        throw triton::exceptions::IrBuilder("IrBuilder::IrBuilder(): The architecture API must be defined.");

      if (symbolicEngine == nullptr)
        throw triton::exceptions::IrBuilder("IrBuilder::IrBuilder(): The symbolic engine API must be defined.");

      if (taintEngine == nullptr)
        throw triton::exceptions::IrBuilder("IrBuilder::IrBuilder(): The taint engines API must be defined.");

      this->architecture              = architecture;
      this->backupSymbolicEngine      = new(std::nothrow) triton::engines::symbolic::SymbolicEngine(architecture, modes, astCtxt, nullptr, true);
      this->symbolicEngine            = symbolicEngine;
      this->taintEngine               = taintEngine;
      this->x86Isa                    = new(std::nothrow) triton::arch::x86::x86Semantics(architecture, symbolicEngine, taintEngine, astCtxt);

      if (this->x86Isa == nullptr || this->backupSymbolicEngine == nullptr)
        throw triton::exceptions::IrBuilder("IrBuilder::IrBuilder(): Not enough memory.");
    }

    IrBuilder::~IrBuilder() {
      delete this->backupSymbolicEngine;
      delete this->x86Isa;
    }

    bool IrBuilder::buildSemantics(triton::arch::Instruction& inst) {
      bool ret = false;

      if (this->architecture->getArchitecture() == triton::arch::ARCH_INVALID)
        throw triton::exceptions::IrBuilder("IrBuilder::buildSemantics(): You must define an architecture.");

      /* Stage 1 - Update the context memory */
      for (auto const& mem : inst.memoryAccess) {
        this->architecture->setConcreteMemoryValue(mem);
      }

      /* Stage 2 - Update the context register */
      for (auto const& reg : inst.registerState) {
        this->architecture->setConcreteRegisterValue(reg.second);
      }

      /* Stage 3 - Initialize the target address of memory operands */
      for (auto& operand : inst.operands) {
        if (operand.getType() == triton::arch::OP_MEM) {
          this->symbolicEngine->initLeaAst(operand.getMemory());
        }
      }

      /* Pre IR processing */
      this->preIrInit(inst);

      /* Processing */
      switch (this->architecture->getArchitecture()) {
        case triton::arch::ARCH_X86:
        case triton::arch::ARCH_X86_64:
          ret = this->x86Isa->buildSemantics(inst);
      }

      /* Post IR processing */
      this->postIrInit(inst);

      return ret;
    }

    void IrBuilder::preIrInit(triton::arch::Instruction& inst) {
      /* Clear previous expressions if exist */
      inst.symbolicExpressions.clear();

      /* Clear implicit and explicit semantics */
      inst.getLoadAccess().clear();
      inst.getReadRegisters().clear();
      inst.getReadImmediates().clear();
      inst.getStoreAccess().clear();
      inst.getWrittenRegisters().clear();

      /* Update instruction address if undefined */
      if (!inst.getAddress())
        inst.setAddress(this->architecture->getConcreteRegisterValue(this->architecture->getParentRegister(ID_REG_IP)).convert_to<triton::uint64>());

      /* Backup the symbolic engine in the case where only the taint is available. */
      if (!this->symbolicEngine->isEnabled()) {
        *this->backupSymbolicEngine = *this->symbolicEngine;
        this->backupAstGarbageCollector = this->astGarbageCollector;
      }
    }

    void IrBuilder::postIrInit(triton::arch::Instruction& inst) {
      std::set<triton::ast::AbstractNode*> uniqueNodes;
      std::vector<triton::engines::symbolic::SymbolicExpression*> newVector;

      auto& loadAccess        = inst.getLoadAccess();
      auto& readRegisters     = inst.getReadRegisters();
      auto& readImmediates    = inst.getReadImmediates();
      auto& storeAccess       = inst.getStoreAccess();
      auto& writtenRegisters  = inst.getWrittenRegisters();

      /* Clear unused data */
      inst.memoryAccess.clear();
      inst.registerState.clear();

      /* Set the taint */
      inst.setTaint();

      // ----------------------------------------------------------------------

      /*
       * If the symbolic engine is disable we delete symbolic
       * expressions and AST nodes. Note that if the taint engine
       * is enable we must compute semanitcs to spread the taint.
       */
      if (!this->symbolicEngine->isEnabled()) {
        /* Clear memory operands */
        this->collectNodes(uniqueNodes, inst.operands, false);

        /* Clear implicit and explicit semantics */
        loadAccess.clear();
        readRegisters.clear();
        readImmediates.clear();
        storeAccess.clear();
        writtenRegisters.clear();

        /* Symbolic Expressions */
        this->removeSymbolicExpressions(inst, uniqueNodes);

        /* Restore backup */
        *this->symbolicEngine = *this->backupSymbolicEngine;
      }

      // ----------------------------------------------------------------------

      /*
       * If the symbolic engine is defined to process symbolic
       * execution only on symbolized expressions, we delete all
       * concrete expressions and their AST nodes.
       */
      if (this->symbolicEngine->isEnabled() && this->modes.isModeEnabled(triton::modes::ONLY_ON_SYMBOLIZED)) {
        /* Clear memory operands */
        this->collectUnsymbolizedNodes(uniqueNodes, inst.operands);

        /* Clear implicit and explicit semantics - MEM */
        this->collectUnsymbolizedNodes(loadAccess);

        /* Clear implicit and explicit semantics - REG */
        this->collectUnsymbolizedNodes(readRegisters);

        /* Clear implicit and explicit semantics - IMM */
        this->collectUnsymbolizedNodes(readImmediates);

        /* Clear implicit and explicit semantics - MEM */
        this->collectUnsymbolizedNodes(storeAccess);

        /* Clear implicit and explicit semantics - REG */
        this->collectUnsymbolizedNodes(writtenRegisters);

        /* Clear symbolic expressions */
        for (auto* se : inst.symbolicExpressions) {
          if (se->isSymbolized() == false) {
            this->astGarbageCollector.extractUniqueAstNodes(uniqueNodes, se->getAst());
            this->symbolicEngine->removeSymbolicExpression(se->getId());
          }
          else
            newVector.push_back(se);
        }
        inst.symbolicExpressions = newVector;
      }

      // ----------------------------------------------------------------------

      /*
       * If the symbolic engine is defined to process symbolic
       * execution only on tainted instructions, we delete all
       * expressions untainted and their AST nodes.
       */
      else if (this->modes.isModeEnabled(triton::modes::ONLY_ON_TAINTED) && !inst.isTainted()) {
        /* Memory operands */
        this->collectNodes(uniqueNodes, inst.operands, true);

        /* Implicit and explicit semantics - MEM */
        this->collectNodes(uniqueNodes, loadAccess);

        /* Implicit and explicit semantics - REG */
        this->collectNodes(uniqueNodes, readRegisters);

        /* Implicit and explicit semantics - IMM */
        this->collectNodes(uniqueNodes, readImmediates);

        /* Implicit and explicit semantics - MEM */
        this->collectNodes(uniqueNodes, storeAccess);

        /* Implicit and explicit semantics - REG */
        this->collectNodes(uniqueNodes, writtenRegisters);

        /* Symbolic Expressions */
        this->removeSymbolicExpressions(inst, uniqueNodes);
      }

      // ----------------------------------------------------------------------

      /* Free collected nodes */
      this->astGarbageCollector.freeAstNodes(uniqueNodes);

      if (!this->symbolicEngine->isEnabled())
        this->astGarbageCollector = this->backupAstGarbageCollector;
    }

    void IrBuilder::removeSymbolicExpressions(triton::arch::Instruction& inst, std::set<triton::ast::AbstractNode*>& uniqueNodes) {
      for (auto const* se : inst.symbolicExpressions) {
        this->astGarbageCollector.extractUniqueAstNodes(uniqueNodes, se->getAst());
        this->symbolicEngine->removeSymbolicExpression(se->getId());
      }
      inst.symbolicExpressions.clear();
    }

    template <typename T>
    void IrBuilder::collectNodes(std::set<triton::ast::AbstractNode*>& uniqueNodes, T& items) const {
      for (auto const& item : items)
        this->astGarbageCollector.extractUniqueAstNodes(uniqueNodes, std::get<1>(item));
      items.clear();
    }

    void IrBuilder::collectNodes(std::set<triton::ast::AbstractNode*>& uniqueNodes, std::vector<triton::arch::OperandWrapper>& operands, bool gc) const {
      for (auto& operand : operands) {
        if (operand.getType() == triton::arch::OP_MEM) {
          if (gc)
            this->astGarbageCollector.extractUniqueAstNodes(uniqueNodes, operand.getMemory().getLeaAst());
          operand.getMemory().setLeaAst(nullptr);
        }
      }
    }

    template <typename T>
    void IrBuilder::collectUnsymbolizedNodes(T& items) const {
      T newItems;

      for (auto const& item : items) {
        if (std::get<1>(item) && std::get<1>(item)->isSymbolized() == true)
          newItems.insert(item);
      }

      items.clear();
      items = newItems;
    }

    void IrBuilder::collectUnsymbolizedNodes(std::set<triton::ast::AbstractNode*>& uniqueNodes, std::vector<triton::arch::OperandWrapper>& operands) const {
      for (auto& operand : operands) {
        if (operand.getType() == triton::arch::OP_MEM) {
          if (operand.getMemory().getLeaAst() && operand.getMemory().getLeaAst()->isSymbolized() == false) {
            this->astGarbageCollector.extractUniqueAstNodes(uniqueNodes, operand.getMemory().getLeaAst());
            operand.getMemory().setLeaAst(nullptr);
          }
        }
      }
    }

  }; /* arch namespace */
}; /* triton namespace */
<|endoftext|>


<commit_before>/* Copyright 2017 The TensorFlow Authors All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/contrib/tpu/profiler/dump_tpu_profile.h"

#include <cstdio>
#include <ctime>
#include <vector>

#include "tensorflow/contrib/tpu/profiler/op_profile.pb.h"
#include "tensorflow/contrib/tpu/profiler/trace_events.pb.h"
#include "tensorflow/contrib/tpu/profiler/trace_events_to_json.h"
#include "tensorflow/core/framework/graph.pb.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/io/compression.h"
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/lib/strings/strcat.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/protobuf/config.pb.h"
#include "tensorflow/core/util/event.pb.h"
#include "tensorflow/core/util/events_writer.h"

namespace tensorflow {
namespace tpu {
namespace {

using ::tensorflow::io::JoinPath;
using ::tensorflow::protobuf::util::JsonOptions;
using ::tensorflow::protobuf::util::MessageToJsonString;

constexpr char kGraphRunPrefix[] = "tpu_profiler.hlo_graph.";
constexpr char kJsonOpProfileFileName[] = "op_profile.json";
constexpr char kJsonTraceFileName[] = "trace.json.gz";
constexpr char kProfilePluginDirectory[] = "plugins/profile/";
constexpr char kProtoTraceFileName[] = "trace";

Status WriteGzippedDataToFile(const string& filename, const string& data) {
  std::unique_ptr<WritableFile> file;
  TF_RETURN_IF_ERROR(Env::Default()->NewWritableFile(filename, &file));
  io::ZlibCompressionOptions options = io::ZlibCompressionOptions::GZIP();
  io::ZlibOutputBuffer buffer(file.get(), options.input_buffer_size,
                              options.output_buffer_size, options);
  TF_RETURN_IF_ERROR(buffer.Init());
  TF_RETURN_IF_ERROR(buffer.Append(data));
  TF_RETURN_IF_ERROR(buffer.Close());
  TF_RETURN_IF_ERROR(file->Close());
  return Status::OK();
}

Status DumpTraceToLogDirectory(StringPiece run_dir, const string& encoded_trace,
                               std::ostream* os) {
  string proto_path = JoinPath(run_dir, kProtoTraceFileName);
  TF_RETURN_IF_ERROR(
      WriteStringToFile(Env::Default(), proto_path, encoded_trace));
  LOG(INFO) << "Dumped raw-proto trace data to " << proto_path;

  string json_path = JoinPath(run_dir, kJsonTraceFileName);
  Trace trace;
  trace.ParseFromString(encoded_trace);
  *os << "Trace contains " << trace.trace_events_size() << " events."
      << std::endl;
  TF_RETURN_IF_ERROR(
      WriteGzippedDataToFile(json_path, TraceEventsToJson(trace)));
  *os << "Dumped JSON trace data to " << json_path << std::endl;
  return Status::OK();
}

Status DumpOpProfileToLogDirectory(StringPiece run_dir,
                                   const tpu::op_profile::Profile& profile,
                                   std::ostream* os) {
  string path = JoinPath(run_dir, kJsonOpProfileFileName);
  string json;
  JsonOptions options;
  options.always_print_primitive_fields = true;
  auto status = MessageToJsonString(profile, &json, options);
  if (!status.ok()) {
    return errors::Internal(
        "Failed to convert op profile to json. Skipping... ",
        string(status.error_message()));
  }
  TF_RETURN_IF_ERROR(WriteStringToFile(Env::Default(), path, json));
  *os << "Dumped json op profile data to " << path << std::endl;
  return Status::OK();
}

Status DumpToolDataToLogDirectory(StringPiece run_dir,
                                  const tensorflow::ProfileToolData& tool,
                                  std::ostream* os) {
  string path = JoinPath(run_dir, tool.name());
  TF_RETURN_IF_ERROR(WriteStringToFile(Env::Default(), path, tool.data()));
  *os << "Dumped tool data for " << tool.name() << " to " << path << std::endl;
  return Status::OK();
}

Status DumpGraphEvents(const string& logdir, const string& run,
                       const ProfileResponse& response, std::ostream* os) {
  int num_graphs = response.computation_graph_size();
  if (response.computation_graph_size() == 0) return Status::OK();
  // The server might generates multiple graphs for one program; we simply
  // pick the first one.
  if (num_graphs > 1) {
    *os << num_graphs
        << " TPU program variants observed over the profiling period. "
        << "One computation graph will be chosen arbitrarily." << std::endl;
  }
  // The graph plugin expects the graph in <logdir>/<run>/<event.file>.
  string run_dir = JoinPath(logdir, strings::StrCat(kGraphRunPrefix, run));
  TF_RETURN_IF_ERROR(Env::Default()->RecursivelyCreateDir(run_dir));
  EventsWriter event_writer(JoinPath(run_dir, "events"));
  Event event;
  // Add the computation graph.
  event.set_graph_def(response.computation_graph(0).SerializeAsString());
  event_writer.WriteEvent(event);
  *os << "Wrote a HLO graph to " << event_writer.FileName() << std::endl;

  if (response.has_hlo_metadata()) {
    tensorflow::TaggedRunMetadata tagged_run_metadata;
    tagged_run_metadata.set_tag(run);
    tagged_run_metadata.set_run_metadata(
        response.hlo_metadata().SerializeAsString());
    tensorflow::Event meta_event;
    *meta_event.mutable_tagged_run_metadata() = tagged_run_metadata;
    event_writer.WriteEvent(meta_event);
    *os << "Wrote HLO ops run metadata to " << event_writer.FileName()
        << std::endl;
  }
  return Status::OK();
}

}  // namespace

Status WriteTensorboardTPUProfile(const string& logdir, const string& run,
                                  const ProfileResponse& response,
                                  std::ostream* os) {
  // Dumps profile data to <logdir>/plugins/profile/<run>/.
  string profile_run_dir = JoinPath(logdir, kProfilePluginDirectory, run);
  TF_RETURN_IF_ERROR(Env::Default()->RecursivelyCreateDir(profile_run_dir));

  // Ignore computation_graph for now.
  const bool empty_trace = response.encoded_trace().empty();
  if (empty_trace) {
    *os << "No trace event is collected." << std::endl;
  } else {
    LOG(INFO) << "Converting trace events to TraceViewer JSON.";
    TF_RETURN_IF_ERROR(
        DumpTraceToLogDirectory(profile_run_dir, response.encoded_trace(), os));
  }
  if (response.has_op_profile() &&
      (response.op_profile().has_by_program_structure() ||
       response.op_profile().has_by_category())) {
    TF_RETURN_IF_ERROR(DumpOpProfileToLogDirectory(profile_run_dir,
                                                   response.op_profile(), os));
  }
  if (!empty_trace && !response.tool_data().empty()) {
    for (const auto& tool_data : response.tool_data()) {
      TF_RETURN_IF_ERROR(
          DumpToolDataToLogDirectory(profile_run_dir, tool_data, os));
    }
  }
  TF_RETURN_IF_ERROR(DumpGraphEvents(logdir, run, response, os));

  return Status::OK();
}

}  // namespace tpu
}  // namespace tensorflow
<commit_msg>internal change<commit_after>/* Copyright 2017 The TensorFlow Authors All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/contrib/tpu/profiler/dump_tpu_profile.h"

#include <cstdio>
#include <ctime>
#include <vector>

#include "tensorflow/contrib/tpu/profiler/op_profile.pb.h"
#include "tensorflow/contrib/tpu/profiler/trace_events.pb.h"
#include "tensorflow/contrib/tpu/profiler/trace_events_to_json.h"
#include "tensorflow/core/framework/graph.pb.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/io/compression.h"
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/lib/strings/strcat.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/protobuf/config.pb.h"
#include "tensorflow/core/util/event.pb.h"
#include "tensorflow/core/util/events_writer.h"

namespace tensorflow {
namespace tpu {
namespace {

using ::tensorflow::io::JoinPath;
using ::tensorflow::protobuf::util::JsonOptions;
using ::tensorflow::protobuf::util::MessageToJsonString;

constexpr char kGraphRunPrefix[] = "tpu_profiler.hlo_graph.";
constexpr char kJsonOpProfileFileName[] = "op_profile.json";
constexpr char kJsonTraceFileName[] = "trace.json.gz";
constexpr char kProfilePluginDirectory[] = "plugins/profile/";
constexpr char kProtoTraceFileName[] = "trace";

Status WriteGzippedDataToFile(const string& filename, const string& data) {
  std::unique_ptr<WritableFile> file;
  TF_RETURN_IF_ERROR(Env::Default()->NewWritableFile(filename, &file));
  io::ZlibCompressionOptions options = io::ZlibCompressionOptions::GZIP();
  io::ZlibOutputBuffer buffer(file.get(), options.input_buffer_size,
                              options.output_buffer_size, options);
  TF_RETURN_IF_ERROR(buffer.Init());
  TF_RETURN_IF_ERROR(buffer.Append(data));
  TF_RETURN_IF_ERROR(buffer.Close());
  TF_RETURN_IF_ERROR(file->Close());
  return Status::OK();
}

Status DumpTraceToLogDirectory(StringPiece run_dir, const string& encoded_trace,
                               std::ostream* os) {
  string proto_path = JoinPath(run_dir, kProtoTraceFileName);
  TF_RETURN_IF_ERROR(
      WriteStringToFile(Env::Default(), proto_path, encoded_trace));
  LOG(INFO) << "Dumped raw-proto trace data to " << proto_path;

  string json_path = JoinPath(run_dir, kJsonTraceFileName);
  Trace trace;
  trace.ParseFromString(encoded_trace);
  *os << "Trace contains " << trace.trace_events_size() << " events."
      << std::endl;
  TF_RETURN_IF_ERROR(
      WriteGzippedDataToFile(json_path, TraceEventsToJson(trace)));
  *os << "Dumped JSON trace data to " << json_path << std::endl;
  return Status::OK();
}

Status DumpOpProfileToLogDirectory(StringPiece run_dir,
                                   const tpu::op_profile::Profile& profile,
                                   std::ostream* os) {
  string path = JoinPath(run_dir, kJsonOpProfileFileName);
  string json;
  JsonOptions options;
  options.always_print_primitive_fields = true;
  auto status = MessageToJsonString(profile, &json, options);
  if (!status.ok()) {
    return errors::Internal(
        "Failed to convert op profile to json. Skipping... ",
        string(status.error_message()));
  }
  TF_RETURN_IF_ERROR(WriteStringToFile(Env::Default(), path, json));
  *os << "Dumped json op profile data to " << path << std::endl;
  return Status::OK();
}

Status DumpToolDataToLogDirectory(StringPiece run_dir,
                                  const tensorflow::ProfileToolData& tool,
                                  std::ostream* os) {
  string path = JoinPath(run_dir, tool.name());
  TF_RETURN_IF_ERROR(WriteStringToFile(Env::Default(), path, tool.data()));
  *os << "Dumped tool data for " << tool.name() << " to " << path << std::endl;
  return Status::OK();
}

Status DumpGraphEvents(const string& logdir, const string& run,
                       const ProfileResponse& response, std::ostream* os) {
  int num_graphs = response.computation_graph_size();
  if (response.computation_graph_size() == 0) return Status::OK();
  // The server might generates multiple graphs for one program; we simply
  // pick the first one.
  if (num_graphs > 1) {
    *os << num_graphs
        << " TPU program variants observed over the profiling period. "
        << "One computation graph will be chosen arbitrarily." << std::endl;
  }
  // The graph plugin expects the graph in <logdir>/<run>/<event.file>.
  string run_dir = JoinPath(logdir, strings::StrCat(kGraphRunPrefix, run));
  TF_RETURN_IF_ERROR(Env::Default()->RecursivelyCreateDir(run_dir));
  EventsWriter event_writer(JoinPath(run_dir, "events"));
  Event event;
  // Add the computation graph.
  event.set_graph_def(response.computation_graph(0).SerializeAsString());
  event_writer.WriteEvent(event);
  *os << "Wrote a HLO graph to " << event_writer.FileName() << std::endl;

  if (response.has_hlo_metadata()) {
    tensorflow::TaggedRunMetadata tagged_run_metadata;
    tagged_run_metadata.set_tag(run);
    tagged_run_metadata.set_run_metadata(
        response.hlo_metadata().SerializeAsString());
    tensorflow::Event meta_event;
    *meta_event.mutable_tagged_run_metadata() = tagged_run_metadata;
    event_writer.WriteEvent(meta_event);
    *os << "Wrote HLO ops run metadata to " << event_writer.FileName()
        << std::endl;
  }
  return Status::OK();
}

}  // namespace

Status WriteTensorboardTPUProfile(const string& logdir, const string& run,
                                  const ProfileResponse& response,
                                  std::ostream* os) {
  // Dumps profile data to <logdir>/plugins/profile/<run>/.
  string profile_run_dir = JoinPath(logdir, kProfilePluginDirectory, run);
  TF_RETURN_IF_ERROR(Env::Default()->RecursivelyCreateDir(profile_run_dir));

  // Ignore computation_graph for now.
  const bool empty_trace = response.encoded_trace().empty();
  if (empty_trace) {
    *os << "No trace event is collected." << std::endl;
  } else {
    LOG(INFO) << "Converting trace events to TraceViewer JSON.";
    TF_RETURN_IF_ERROR(
        DumpTraceToLogDirectory(profile_run_dir, response.encoded_trace(), os));
  }
  if (response.has_op_profile() &&
      (response.op_profile().has_by_program_structure() ||
       response.op_profile().has_by_category())) {
    TF_RETURN_IF_ERROR(DumpOpProfileToLogDirectory(profile_run_dir,
                                                   response.op_profile(), os));
  }
  if (!empty_trace && !response.tool_data().empty()) {
    for (const auto& tool_data : response.tool_data()) {
      TF_RETURN_IF_ERROR(
          DumpToolDataToLogDirectory(profile_run_dir, tool_data, os));
    }
  }

  return Status::OK();
}

}  // namespace tpu
}  // namespace tensorflow
<|endoftext|>


<commit_before>//---------------------------------------------------------------------------
/// \file   es/component.hpp
/// \brief  A component is a data type that can be assigned to entities
//
// Copyright 2013, nocte@hippie.nu            Released under the MIT License.
//---------------------------------------------------------------------------
#pragma once

#include <memory>
#include <string>
#include <typeinfo>

namespace es {

class storage;

/** A component is a data type that can be assigned to entities.
 * For example, an entity could have a position and a velocity.  The position
 * would be a component, and the data type would be a 2- or 3-dimensional
 * coordinate.  The velocity would be yet another component, and the data
 * type would be a vector. */
class component
{
    friend class storage;

protected:
    /** Placeholder for complex data types.
     *  Some data types don't have a fixed, flat memory layout.  This
     *  class defines an interface that can be used as a placeholder in
     *  the entity buffer, while the actual object resides on the heap. */
    class placeholder
    {
    public:
        virtual ~placeholder() { }

        /** Return a copy of the underlying object. */
        virtual placeholder* clone() const = 0;
    };

public:
    /** @param name     Descriptive name
     *  @param size     Size of an instance of this component in bytes.  For
     *                  components that need a placeholder, this should be
     *                  sizeof(placeholder).
     *  @param ph       Simple types should pass a nullptr here.  Complex types
     *                  should pass a pointer to a placeholder instance of
     *                  the correct type. */
    component(std::string name, size_t size, std::unique_ptr<placeholder> ph)
        : name_(std::move(name))
        , size_(size)
        , ph_(std::move(ph))
    { }

    const std::string&    name() const      { return name_; }
    size_t                size() const      { return size_; }
    bool                  is_flat() const   { return ph_ == nullptr; }

    bool operator== (const std::string& compare) const
        { return name_ == compare; }

    bool operator!= (const std::string& compare) const
        { return name_ != compare; }

private:
    std::string             name_;
    size_t                  size_;
    std::unique_ptr<placeholder> ph_;
};

} // namespace es

<commit_msg>Explicit move constructor for VS2013<commit_after>//---------------------------------------------------------------------------
/// \file   es/component.hpp
/// \brief  A component is a data type that can be assigned to entities
//
// Copyright 2013, nocte@hippie.nu            Released under the MIT License.
//---------------------------------------------------------------------------
#pragma once

#include <memory>
#include <string>
#include <typeinfo>

namespace es {

class storage;

/** A component is a data type that can be assigned to entities.
 * For example, an entity could have a position and a velocity.  The position
 * would be a component, and the data type would be a 2- or 3-dimensional
 * coordinate.  The velocity would be yet another component, and the data
 * type would be a vector. */
class component
{
    friend class storage;

protected:
    /** Placeholder for complex data types.
     *  Some data types don't have a fixed, flat memory layout.  This
     *  class defines an interface that can be used as a placeholder in
     *  the entity buffer, while the actual object resides on the heap. */
    class placeholder
    {
    public:
        virtual ~placeholder() { }

        /** Return a copy of the underlying object. */
        virtual placeholder* clone() const = 0;
    };

public:
    /** @param name     Descriptive name
     *  @param size     Size of an instance of this component in bytes.  For
     *                  components that need a placeholder, this should be
     *                  sizeof(placeholder).
     *  @param ph       Simple types should pass a nullptr here.  Complex types
     *                  should pass a pointer to a placeholder instance of
     *                  the correct type. */
    component(std::string name, size_t size, std::unique_ptr<placeholder> ph)
        : name_(std::move(name))
        , size_(size)
        , ph_(std::move(ph))
    { }

	component(component&& m)
		: name_(std::move(m.name_))
		, size_(m.size_)
		, ph_(std::move(m.ph_))
	{
		m.size_ = 0;
	}

    const std::string&    name() const      { return name_; }
    size_t                size() const      { return size_; }
    bool                  is_flat() const   { return ph_ == nullptr; }

    bool operator== (const std::string& compare) const
        { return name_ == compare; }

    bool operator!= (const std::string& compare) const
        { return name_ != compare; }

private:
    std::string             name_;
    size_t                  size_;
    std::unique_ptr<placeholder> ph_;
};

} // namespace es

<|endoftext|>


<commit_before>// Copyright 2019 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "tensorflow_text/core/kernels/wordpiece_tokenizer.h"

#include "absl/strings/str_cat.h"
#include "absl/strings/str_join.h"
#include "absl/strings/string_view.h"
#include "absl/strings/string_view_utils.h"
#include "icu4c/source/common/unicode/schriter.h"
#include "icu4c/source/common/unicode/unistr.h"
#include "icu4c/source/common/unicode/utf8.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/lib/core/errors.h"

namespace tensorflow {
namespace text {

constexpr int64 kOutOfVocabValue = -1;

LookupTableVocab::LookupTableVocab(lookup::LookupInterface* table,
                                   OpKernelContext* ctx)
    : table_(table), ctx_(ctx), default_value_(DT_INT64, TensorShape({1})) {
  default_value_.flat<int64>()(0) = kOutOfVocabValue;
}

Status LookupTableVocab::Contains(const string& key, bool* value) {
  if (value == nullptr) {
    return errors::InvalidArgument("Bad 'value' param.");
  }
  Tensor keys(DT_STRING, TensorShape({1}));
  keys.flat<string>()(0) = key;
  Tensor values(DT_INT64, TensorShape({1}));
  TF_RETURN_IF_ERROR(table_->Find(ctx_, keys, &values, default_value_));

  if (static_cast<int64>(values.flat<int64>()(0)) != kOutOfVocabValue) {
    *value = true;
    return Status::OK();
  }
  *value = false;
  return Status::OK();
}

namespace {

Status Lookup(int byte_start, int byte_end, const string& token,
              const string& suffix_indicator, LookupTableVocab* vocab_map,
              bool* in_vocab) {
  int byte_len = byte_end - byte_start;
  absl::string_view substr(token.data() + byte_start, byte_len);
  string lookup_value;
  if (byte_start > 0) {
    lookup_value = absl::StrCat(suffix_indicator, substr);
  } else {
    absl::CopyToString(substr, &lookup_value);
  }
  return vocab_map->Contains(lookup_value, in_vocab);
}

// Sets byte_end to the longest byte sequence which:
// 1) is a proper UTF8 sequence
// 2) is in the vocab
// If no match is found, found_match is set to false.
Status LongestMatchStartingAt(int byte_start, const string& token,
                              const string& suffix_indicator,
                              LookupTableVocab* vocab_map, int* byte_end,
                              bool* found_match) {
  const char* token_bytes = token.data();
  const int token_len = token.length();
  std::vector<int32_t> byte_ends;
  for (int32_t i = byte_start; i < token_len;) {
    UChar32 c;
    U8_NEXT(token_bytes, i, token_len, c);
    byte_ends.push_back(i);
  }
  int n = byte_ends.size();
  for (int i = n - 1; i >= 0; i--) {
    bool in_vocab;
    TF_RETURN_IF_ERROR(Lookup(byte_start, byte_ends[i], token, suffix_indicator,
                              vocab_map, &in_vocab));
    if (in_vocab) {
      *byte_end = byte_ends[i];
      *found_match = true;
      return Status::OK();
    }
  }
  *found_match = false;
  return Status::OK();
}

// Sets the outputs 'begin_offset', 'end_offset' and 'num_word_pieces' when no
// token is found.
Status NoTokenFound(const string& token, bool use_unknown_token,
                    const string& unknown_token, std::vector<string>* subwords,
                    std::vector<int>* begin_offset,
                    std::vector<int>* end_offset, int* num_word_pieces) {
  begin_offset->push_back(0);
  if (use_unknown_token) {
    subwords->push_back(unknown_token);
    end_offset->push_back(token.length());
  } else {
    subwords->push_back(token);
    end_offset->push_back(token.length());
  }
  ++(*num_word_pieces);

  return Status::OK();
}

// When a subword is found, this helper function will add the outputs to
// 'subwords', 'begin_offset' and 'end_offset'.
void AddWord(const string& token, int byte_start, int byte_end,
             const string& suffix_indicator, std::vector<string>* subwords,
             std::vector<int>* begin_offset, std::vector<int>* end_offset) {
  begin_offset->push_back(byte_start);
  int len = byte_end - byte_start;
  if (byte_start > 0) {
    // Prepend suffix_indicator if the token is within a word.
    subwords->push_back(
        ::absl::StrCat(suffix_indicator, token.substr(byte_start, len)));
  } else {
    subwords->push_back(token.substr(byte_start, len));
  }
  end_offset->push_back(byte_end);
}

Status TokenizeL2RGreedy(const string& token, const int64 max_bytes_per_token,
                         const string& suffix_indicator, bool use_unknown_token,
                         const string& unknown_token,
                         LookupTableVocab* vocab_map,
                         std::vector<string>* subwords,
                         std::vector<int>* begin_offset,
                         std::vector<int>* end_offset, int* num_word_pieces) {
  std::vector<string> candidate_subwords;
  std::vector<int> candidate_begin_offsets;
  std::vector<int> candidate_end_offsets;
  const int token_len = token.length();
  for (int byte_start = 0; byte_start < token_len;) {
    int byte_end;
    bool found_subword;
    TF_RETURN_IF_ERROR(LongestMatchStartingAt(byte_start, token,
                                              suffix_indicator, vocab_map,
                                              &byte_end, &found_subword));
    if (found_subword) {
      AddWord(token, byte_start, byte_end, suffix_indicator,
              &candidate_subwords, &candidate_begin_offsets,
              &candidate_end_offsets);
      byte_start = byte_end;
    } else {
      return NoTokenFound(token, use_unknown_token, unknown_token, subwords,
                          begin_offset, end_offset, num_word_pieces);
    }
  }

  subwords->insert(subwords->end(), candidate_subwords.begin(),
                   candidate_subwords.end());
  begin_offset->insert(begin_offset->end(), candidate_begin_offsets.begin(),
                       candidate_begin_offsets.end());
  end_offset->insert(end_offset->end(), candidate_end_offsets.begin(),
                     candidate_end_offsets.end());
  *num_word_pieces += candidate_subwords.size();
  return Status::OK();
}

}  // namespace

Status WordpieceTokenize(const string& token, const int64 max_bytes_per_token,
                         const string& suffix_indicator, bool use_unknown_token,
                         const string& unknown_token,
                         LookupTableVocab* vocab_map,
                         std::vector<string>* subwords,
                         std::vector<int>* begin_offset,
                         std::vector<int>* end_offset, int* num_word_pieces) {
  int token_len = token.size();
  if (token_len > max_bytes_per_token) {
    begin_offset->push_back(0);
    *num_word_pieces = 1;
    if (use_unknown_token) {
      end_offset->push_back(unknown_token.size());
      subwords->emplace_back(unknown_token);
    } else {
      subwords->emplace_back(token);
      end_offset->push_back(token.size());
    }
    return Status::OK();
  }
  return TokenizeL2RGreedy(token, max_bytes_per_token, suffix_indicator,
                           use_unknown_token, unknown_token, vocab_map,
                           subwords, begin_offset, end_offset, num_word_pieces);
}

}  // namespace text
}  // namespace tensorflow
<commit_msg>Remove non-public absl dependency<commit_after>// Copyright 2019 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "tensorflow_text/core/kernels/wordpiece_tokenizer.h"

#include "absl/strings/str_cat.h"
#include "absl/strings/str_join.h"
#include "absl/strings/string_view.h"
#include "icu4c/source/common/unicode/schriter.h"
#include "icu4c/source/common/unicode/unistr.h"
#include "icu4c/source/common/unicode/utf8.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/lib/core/errors.h"

namespace tensorflow {
namespace text {

constexpr int64 kOutOfVocabValue = -1;

LookupTableVocab::LookupTableVocab(lookup::LookupInterface* table,
                                   OpKernelContext* ctx)
    : table_(table), ctx_(ctx), default_value_(DT_INT64, TensorShape({1})) {
  default_value_.flat<int64>()(0) = kOutOfVocabValue;
}

Status LookupTableVocab::Contains(const string& key, bool* value) {
  if (value == nullptr) {
    return errors::InvalidArgument("Bad 'value' param.");
  }
  Tensor keys(DT_STRING, TensorShape({1}));
  keys.flat<string>()(0) = key;
  Tensor values(DT_INT64, TensorShape({1}));
  TF_RETURN_IF_ERROR(table_->Find(ctx_, keys, &values, default_value_));

  if (static_cast<int64>(values.flat<int64>()(0)) != kOutOfVocabValue) {
    *value = true;
    return Status::OK();
  }
  *value = false;
  return Status::OK();
}

namespace {

Status Lookup(int byte_start, int byte_end, const string& token,
              const string& suffix_indicator, LookupTableVocab* vocab_map,
              bool* in_vocab) {
  int byte_len = byte_end - byte_start;
  absl::string_view substr(token.data() + byte_start, byte_len);
  string lookup_value;
  if (byte_start > 0) {
    lookup_value = absl::StrCat(suffix_indicator, substr);
  } else {
    // absl::CopyToString
    lookup_value.assign(substr.begin(), substr.end());
  }
  return vocab_map->Contains(lookup_value, in_vocab);
}

// Sets byte_end to the longest byte sequence which:
// 1) is a proper UTF8 sequence
// 2) is in the vocab
// If no match is found, found_match is set to false.
Status LongestMatchStartingAt(int byte_start, const string& token,
                              const string& suffix_indicator,
                              LookupTableVocab* vocab_map, int* byte_end,
                              bool* found_match) {
  const char* token_bytes = token.data();
  const int token_len = token.length();
  std::vector<int32_t> byte_ends;
  for (int32_t i = byte_start; i < token_len;) {
    UChar32 c;
    U8_NEXT(token_bytes, i, token_len, c);
    byte_ends.push_back(i);
  }
  int n = byte_ends.size();
  for (int i = n - 1; i >= 0; i--) {
    bool in_vocab;
    TF_RETURN_IF_ERROR(Lookup(byte_start, byte_ends[i], token, suffix_indicator,
                              vocab_map, &in_vocab));
    if (in_vocab) {
      *byte_end = byte_ends[i];
      *found_match = true;
      return Status::OK();
    }
  }
  *found_match = false;
  return Status::OK();
}

// Sets the outputs 'begin_offset', 'end_offset' and 'num_word_pieces' when no
// token is found.
Status NoTokenFound(const string& token, bool use_unknown_token,
                    const string& unknown_token, std::vector<string>* subwords,
                    std::vector<int>* begin_offset,
                    std::vector<int>* end_offset, int* num_word_pieces) {
  begin_offset->push_back(0);
  if (use_unknown_token) {
    subwords->push_back(unknown_token);
    end_offset->push_back(token.length());
  } else {
    subwords->push_back(token);
    end_offset->push_back(token.length());
  }
  ++(*num_word_pieces);

  return Status::OK();
}

// When a subword is found, this helper function will add the outputs to
// 'subwords', 'begin_offset' and 'end_offset'.
void AddWord(const string& token, int byte_start, int byte_end,
             const string& suffix_indicator, std::vector<string>* subwords,
             std::vector<int>* begin_offset, std::vector<int>* end_offset) {
  begin_offset->push_back(byte_start);
  int len = byte_end - byte_start;
  if (byte_start > 0) {
    // Prepend suffix_indicator if the token is within a word.
    subwords->push_back(
        ::absl::StrCat(suffix_indicator, token.substr(byte_start, len)));
  } else {
    subwords->push_back(token.substr(byte_start, len));
  }
  end_offset->push_back(byte_end);
}

Status TokenizeL2RGreedy(const string& token, const int64 max_bytes_per_token,
                         const string& suffix_indicator, bool use_unknown_token,
                         const string& unknown_token,
                         LookupTableVocab* vocab_map,
                         std::vector<string>* subwords,
                         std::vector<int>* begin_offset,
                         std::vector<int>* end_offset, int* num_word_pieces) {
  std::vector<string> candidate_subwords;
  std::vector<int> candidate_begin_offsets;
  std::vector<int> candidate_end_offsets;
  const int token_len = token.length();
  for (int byte_start = 0; byte_start < token_len;) {
    int byte_end;
    bool found_subword;
    TF_RETURN_IF_ERROR(LongestMatchStartingAt(byte_start, token,
                                              suffix_indicator, vocab_map,
                                              &byte_end, &found_subword));
    if (found_subword) {
      AddWord(token, byte_start, byte_end, suffix_indicator,
              &candidate_subwords, &candidate_begin_offsets,
              &candidate_end_offsets);
      byte_start = byte_end;
    } else {
      return NoTokenFound(token, use_unknown_token, unknown_token, subwords,
                          begin_offset, end_offset, num_word_pieces);
    }
  }

  subwords->insert(subwords->end(), candidate_subwords.begin(),
                   candidate_subwords.end());
  begin_offset->insert(begin_offset->end(), candidate_begin_offsets.begin(),
                       candidate_begin_offsets.end());
  end_offset->insert(end_offset->end(), candidate_end_offsets.begin(),
                     candidate_end_offsets.end());
  *num_word_pieces += candidate_subwords.size();
  return Status::OK();
}

}  // namespace

Status WordpieceTokenize(const string& token, const int64 max_bytes_per_token,
                         const string& suffix_indicator, bool use_unknown_token,
                         const string& unknown_token,
                         LookupTableVocab* vocab_map,
                         std::vector<string>* subwords,
                         std::vector<int>* begin_offset,
                         std::vector<int>* end_offset, int* num_word_pieces) {
  int token_len = token.size();
  if (token_len > max_bytes_per_token) {
    begin_offset->push_back(0);
    *num_word_pieces = 1;
    if (use_unknown_token) {
      end_offset->push_back(unknown_token.size());
      subwords->emplace_back(unknown_token);
    } else {
      subwords->emplace_back(token);
      end_offset->push_back(token.size());
    }
    return Status::OK();
  }
  return TokenizeL2RGreedy(token, max_bytes_per_token, suffix_indicator,
                           use_unknown_token, unknown_token, vocab_map,
                           subwords, begin_offset, end_offset, num_word_pieces);
}

}  // namespace text
}  // namespace tensorflow
<|endoftext|>


<commit_before>/* See LICENSE.

   The Minor Planet Center accepts astrometry from spacecraft using
a modification of their usual 80-column "punched-card" format.  A
second line is used to tell you where the spacecraft was,  relative
to the geocenter.

https://minorplanetcenter.net/iau/info/SatelliteObs.html

   This code can reset those positions for Earth-orbiting spacecraft using
TLEs ('two-line elements';  https://www.projectpluto.com/tle_info.htm.)
This helps when accurate positions are not provided or completely trusted
or appear to be bad.

   This code will read the 80-column astrometry and,  when it finds a
spacecraft position report (the "second line"),  look through the TLE
file for a matching TLE.  If it finds one,  it'll compute the spacecraft
location for that time and modify the position accordingly.

   This was written specifically to handle a problem with a satellite in
LEO.  TLEs don't exist for heliocentric objects,  but I may revise this
code eventually to add positions for them (this could be done by,  for
example,  requesting them from JPL Horizons).  TLEs exist for TESS,
computed by me,  but some work would be required to make those actually
function properly (the numbers are larger and the format is somewhat
different as a result).  So this really works,  at present,  only for
(C51) WISE, (C52) Swift,  and (C53) NEOSSat.             */

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <assert.h>
#include <math.h>
#include "norad.h"
#include "watdefs.h"
#include "afuncs.h"
#include "mpc_func.h"

#define PI \
   3.1415926535897932384626433832795028841971693993751058209749445923

int find_tle( tle_t *tle, const char *filename, const int norad_no)
{
   FILE *ifile = fopen( filename, "rb");
   char line0[100], line1[100], line2[100];
   int rval = -1;

   if( !ifile)
      {
      fprintf( stderr, "Couldn't open TLE file '%s'\n", filename);
      exit( -1);
      }
   *line0 = *line1 = '\0';
   while( rval && fgets( line2, sizeof( line2), ifile))
      {
      if( *line1 == '1' && *line2 == '2' && atoi( line1 + 2) == norad_no)
         if( parse_elements( line1, line2, tle) >= 0)
            rval = 0;
      strcpy( line0, line1);
      strcpy( line1, line2);
      }
   fclose( ifile);
   if( rval)
      {
      fprintf( stderr, "Couldn't find TLEs for %5d in TLE file '%s'\n",
                           norad_no, filename);
      exit( -1);
      }
   return( rval);
}

/*
C49 = 29510 = 2006-047A = STEREO-A
C50 = 29511 = 2006-047B = STEREO-B
C51 = 36119 = 2009-071A = WISE   *
C52 = 28485 = 2004-047A = Swift   *
C53 = 39089 = 2013-009D = NEOSSat  *
C54 = 28928 = 2006-001A = New Horizons
C55 = 34380 = 2009-011A = Kepler
C56 = 41043 = 2015-070A = LISA-Pathfinder
C57 = 43435 = 2018-038A = TESS      *
   Note that only the asterisked objects are actually in earth orbit
and therefore have TLEs.  But I may try to figure out some way to add
in positions for the heliocentric spacecraft later.  */

static int mpc_code_to_norad_number( const char *mpc_code)
{
   const char *codes = "C49 C50 C51 C52 C53 C54 C55 C56 C57 ";
   const int norad_numbers[] = { 29510, 29511, 36119, 28485, 39089,
                                        28928, 34380, 41043, 43435 };
   int i;

   for( i = 0; codes[i * 4]; i++)
      if( !memcmp( codes + i * 4, mpc_code, 3))
         return( norad_numbers[i]);
   assert( 1);          /* not supposed to ever happen,  unless a new */
   return( 0);          /* satellite has been added */
}

int main( const int argc, const char **argv)
{
   FILE *ifile;
   const char *tle_filename = (argc > 2 ? argv[2] : "all_tle.txt");
   const char *astrometry_filename = argv[1];
   int is_deep = 0, curr_norad = 0;
   char buff[200];
   double params[N_SAT_PARAMS];
   const double J2000 = 2451545.;
   tle_t tle;

   ifile = fopen( astrometry_filename, "rb");
   assert( ifile);
   while( fgets( buff, sizeof( buff), ifile))
      {
      double jd;

      if( strlen( buff) > 80 && buff[14] == 's'
            && (jd = extract_date_from_mpc_report( buff, NULL)) > 2400000.)
         {
         double state[6], state_j2000[6], precess_matrix[9];
         double t_since;
         const int norad_number = mpc_code_to_norad_number( buff + 77);
         int i;

         if( curr_norad != norad_number)
            {
            curr_norad = norad_number;
            find_tle( &tle, tle_filename, norad_number);
            is_deep = select_ephemeris( &tle);
            if( is_deep)
               SDP4_init( params, &tle);
            else
               SGP4_init( params, &tle);
            }
         t_since = (jd - tle.epoch) * minutes_per_day;
         if( is_deep)
            SDP4( t_since, &tle, params, state, state + 3);
         else
            SGP4( t_since, &tle, params, state, state + 3);
         setup_precession( precess_matrix, 2000. + (jd - J2000) / 365.25, 2000.);
         precess_vector( precess_matrix, state, state_j2000);
         precess_vector( precess_matrix, state + 3, state_j2000 + 3);
         for( i = 0; i < 3; i++)
            {
            char *tptr = buff + 34 + i * 12;

            sprintf( tptr, "%11.4f", fabs( state_j2000[i]));
            *tptr = (state_j2000[i] > 0. ? '+' : '-');
            tptr[11] = ' ';
            }
         }
      printf( "%s", buff);
      }
   fclose( ifile);
   return( 0);
}
<commit_msg>'line2' : improved error handling and message.<commit_after>/* See LICENSE.

   The Minor Planet Center accepts astrometry from spacecraft using
a modification of their usual 80-column "punched-card" format.  A
second line is used to tell you where the spacecraft was,  relative
to the geocenter.

https://minorplanetcenter.net/iau/info/SatelliteObs.html

   This code can reset those positions for Earth-orbiting spacecraft using
TLEs ('two-line elements';  https://www.projectpluto.com/tle_info.htm.)
This helps when accurate positions are not provided or completely trusted
or appear to be bad.

   This code will read the 80-column astrometry and,  when it finds a
spacecraft position report (the "second line"),  look through the TLE
file for a matching TLE.  If it finds one,  it'll compute the spacecraft
location for that time and modify the position accordingly.

   This was written specifically to handle a problem with a satellite in
LEO.  TLEs don't exist for heliocentric objects,  but I may revise this
code eventually to add positions for them (this could be done by,  for
example,  requesting them from JPL Horizons).  TLEs exist for TESS,
computed by me,  but some work would be required to make those actually
function properly (the numbers are larger and the format is somewhat
different as a result).  So this really works,  at present,  only for
(C51) WISE, (C52) Swift,  and (C53) NEOSSat.             */

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <assert.h>
#include <errno.h>
#include <math.h>
#include "norad.h"
#include "watdefs.h"
#include "afuncs.h"
#include "mpc_func.h"

#define PI \
   3.1415926535897932384626433832795028841971693993751058209749445923

int find_tle( tle_t *tle, const char *filename, const int norad_no)
{
   FILE *ifile = fopen( filename, "rb");
   char line0[100], line1[100], line2[100];
   int rval = -1;

   if( !ifile)
      {
      fprintf( stderr, "Couldn't open TLE file '%s'\n", filename);
      exit( -1);
      }
   *line0 = *line1 = '\0';
   while( rval && fgets( line2, sizeof( line2), ifile))
      {
      if( *line1 == '1' && *line2 == '2' && atoi( line1 + 2) == norad_no)
         if( parse_elements( line1, line2, tle) >= 0)
            rval = 0;
      strcpy( line0, line1);
      strcpy( line1, line2);
      }
   fclose( ifile);
   if( rval)
      {
      fprintf( stderr, "Couldn't find TLEs for %5d in TLE file '%s'\n",
                           norad_no, filename);
      exit( -1);
      }
   return( rval);
}

/*
C49 = 29510 = 2006-047A = STEREO-A
C50 = 29511 = 2006-047B = STEREO-B
C51 = 36119 = 2009-071A = WISE   *
C52 = 28485 = 2004-047A = Swift   *
C53 = 39089 = 2013-009D = NEOSSat  *
C54 = 28928 = 2006-001A = New Horizons
C55 = 34380 = 2009-011A = Kepler
C56 = 41043 = 2015-070A = LISA-Pathfinder
C57 = 43435 = 2018-038A = TESS      *
   Note that only the asterisked objects are actually in earth orbit
and therefore have TLEs.  But I may try to figure out some way to add
in positions for the heliocentric spacecraft later.  */

static int mpc_code_to_norad_number( const char *mpc_code)
{
   const char *codes = "C49 C50 C51 C52 C53 C54 C55 C56 C57 ";
   const int norad_numbers[] = { 29510, 29511, 36119, 28485, 39089,
                                        28928, 34380, 41043, 43435 };
   int i;

   for( i = 0; codes[i * 4]; i++)
      if( !memcmp( codes + i * 4, mpc_code, 3))
         return( norad_numbers[i]);
   assert( 1);          /* not supposed to ever happen,  unless a new */
   return( 0);          /* satellite has been added */
}

int main( const int argc, const char **argv)
{
   const char *tle_filename = (argc > 2 ? argv[2] : "all_tle.txt");
   const char *astrometry_filename = argv[1];
   FILE *ifile = (argc > 1 ? fopen( astrometry_filename, "rb") : NULL);
   int is_deep = 0, curr_norad = 0;
   char buff[200];
   double params[N_SAT_PARAMS];
   const double J2000 = 2451545.;
   tle_t tle;

   if( argc > 1 && !ifile)
      fprintf( stderr, "'%s' not found : %s\n", astrometry_filename, strerror( errno));
   if( !ifile)
      {
      fprintf( stderr, "'line2' takes as a command-line argument the name of the input\n"
                       "astrometry file.  It sends astrometry with corrected/added\n"
                       "spacecraft locations to stdout.\n");
      exit( -1);
      }
   while( fgets( buff, sizeof( buff), ifile))
      {
      double jd;

      if( strlen( buff) > 80 && buff[14] == 's'
            && (jd = extract_date_from_mpc_report( buff, NULL)) > 2400000.)
         {
         double state[6], state_j2000[6], precess_matrix[9];
         double t_since;
         const int norad_number = mpc_code_to_norad_number( buff + 77);
         int i;

         if( curr_norad != norad_number)
            {
            curr_norad = norad_number;
            find_tle( &tle, tle_filename, norad_number);
            is_deep = select_ephemeris( &tle);
            if( is_deep)
               SDP4_init( params, &tle);
            else
               SGP4_init( params, &tle);
            }
         t_since = (jd - tle.epoch) * minutes_per_day;
         if( is_deep)
            SDP4( t_since, &tle, params, state, state + 3);
         else
            SGP4( t_since, &tle, params, state, state + 3);
         setup_precession( precess_matrix, 2000. + (jd - J2000) / 365.25, 2000.);
         precess_vector( precess_matrix, state, state_j2000);
         precess_vector( precess_matrix, state + 3, state_j2000 + 3);
         for( i = 0; i < 3; i++)
            {
            char *tptr = buff + 34 + i * 12;

            sprintf( tptr, "%11.4f", fabs( state_j2000[i]));
            *tptr = (state_j2000[i] > 0. ? '+' : '-');
            tptr[11] = ' ';
            }
         }
      printf( "%s", buff);
      }
   fclose( ifile);
   return( 0);
}
<|endoftext|>