<commit_before>// Copyright (C) 2018 Elviss Strazdins
// This file is part of the Ouzel engine.

#include "core/Setup.h"

#if OUZEL_COMPILE_OPENAL

#if OUZEL_PLATFORM_IOS || OUZEL_PLATFORM_TVOS
#include <objc/message.h>
extern "C" id const AVAudioSessionCategoryAmbient;
#endif

#include "AudioDeviceAL.hpp"
#include "core/Engine.hpp"
#include "utils/Errors.hpp"
#include "utils/Log.hpp"

namespace ouzel
{
    namespace audio
    {
        AudioDeviceAL::AudioDeviceAL():
            AudioDevice(Audio::Driver::OPENAL)
        {
#if OUZEL_PLATFORM_IOS || OUZEL_PLATFORM_TVOS
            id audioSession = reinterpret_cast<id (*)(Class, SEL)>(&objc_msgSend)(objc_getClass("AVAudioSession"), sel_getUid("sharedInstance"));
            reinterpret_cast<BOOL (*)(id, SEL, id, id)>(&objc_msgSend)(audioSession, sel_getUid("setCategory:error:"), AVAudioSessionCategoryAmbient, nil);
#endif

#if OUZEL_MULTITHREADED
            running = false;
#endif

            std::fill(std::begin(bufferIds), std::end(bufferIds), 0);
        }

        AudioDeviceAL::~AudioDeviceAL()
        {
#if OUZEL_MULTITHREADED
            running = false;
            if (audioThread.isJoinable()) audioThread.join();
#endif

            if (sourceId)
            {
                alSourceStop(sourceId);
                alSourcei(sourceId, AL_BUFFER, 0);
                alDeleteSources(1, &sourceId);

                if (checkOpenALError())
                    Log(Log::Level::ERR) << "Failed to delete OpenAL source";
            }

            for (ALuint bufferId : bufferIds)
            {
                if (bufferId)
                {
                    alDeleteBuffers(1, &bufferId);

                    if (checkOpenALError())
                        Log(Log::Level::ERR) << "Failed to delete OpenAL buffer";
                }
            }

            if (context)
            {
                alcMakeContextCurrent(nullptr);
                alcDestroyContext(context);
            }

            if (device)
                alcCloseDevice(device);
        }

        void AudioDeviceAL::init(bool debugAudio)
        {
            AudioDevice::init(debugAudio);

            const ALCchar* deviceName = alcGetString(nullptr, ALC_DEFAULT_DEVICE_SPECIFIER);

            if (deviceName)
                Log(Log::Level::INFO) << "Using " << deviceName << " for audio";

            device = alcOpenDevice(deviceName);

            if (!device || checkALCError())
                throw SystemError("Failed to create OpenAL device");

            int capabilities[] =
            {
                ALC_FREQUENCY, 44100,
                ALC_STEREO_SOURCES, 4,
                0, 0
            };

            context = alcCreateContext(device, capabilities);

            if (checkALCError())
                throw SystemError("Failed to create OpenAL context");

            alcMakeContextCurrent(context);

            if (checkALCError())
                throw SystemError("Failed to make OpenAL context current");

#if !OUZEL_PLATFORM_EMSCRIPTEN
            format40 = alGetEnumValue("AL_FORMAT_QUAD16");
            format51 = alGetEnumValue("AL_FORMAT_51CHN16");
            format61 = alGetEnumValue("AL_FORMAT_61CHN16");
            format71 = alGetEnumValue("AL_FORMAT_71CHN16");
#endif

            if (checkOpenALError())
                Log(Log::Level::WARN) << "Failed to get OpenAL enum values";

            alGenSources(1, &sourceId);

            if (checkOpenALError())
                throw SystemError("Failed to create OpenAL source");

            alGenBuffers(2, bufferIds);

            if (checkOpenALError())
                throw SystemError("Failed to create OpenAL buffers");

            switch (channels)
            {
                case 1: format = AL_FORMAT_MONO16; break;
                case 2: format = AL_FORMAT_STEREO16; break;
                case 4: format = format40; break;
                case 6: format = format51; break;
                case 7: format = format61; break;
                case 8: format = format71; break;
                default:
                    throw SystemError("Invalid channel count");
            }

            sampleFormat = Audio::SampleFormat::SINT16;

            getData(bufferSize / (channels * sizeof(int16_t)), data);

            alBufferData(bufferIds[0], format,
                         data.data(),
                         static_cast<ALsizei>(data.size()),
                         static_cast<ALsizei>(sampleRate));

            getData(bufferSize / (channels * sizeof(int16_t)), data);

            alBufferData(bufferIds[1], format,
                         data.data(),
                         static_cast<ALsizei>(data.size()),
                         static_cast<ALsizei>(sampleRate));

            nextBuffer = 0;

            alSourceQueueBuffers(sourceId, 2, bufferIds);

            if (checkOpenALError())
                throw SystemError("Failed to queue OpenAL buffers");

            alSourcePlay(sourceId);

            if (checkOpenALError())
                throw SystemError("Failed to play OpenAL source");

#if OUZEL_MULTITHREADED
            running = true;
            audioThread = Thread(std::bind(&AudioDeviceAL::run, this), "Audio");
#endif
        }

        void AudioDeviceAL::process()
        {
            AudioDevice::process();

            alcMakeContextCurrent(context);

            if (checkALCError())
                throw SystemError("Failed to make OpenAL context current");

            ALint buffersProcessed;
            alGetSourcei(sourceId, AL_BUFFERS_PROCESSED, &buffersProcessed);

            if (checkOpenALError())
                throw SystemError("Failed to get processed buffer count");

            // requeue all processed buffers
            for (; buffersProcessed > 0; --buffersProcessed)
            {
                alSourceUnqueueBuffers(sourceId, 1, &bufferIds[nextBuffer]);

                if (checkOpenALError())
                    throw SystemError("Failed to unqueue OpenAL buffer");

                if (!getData(bufferSize / (channels * sizeof(int16_t)), data))
                    throw SystemError("Failed to get data");

                alBufferData(bufferIds[nextBuffer], format,
                             data.data(),
                             static_cast<ALsizei>(data.size()),
                             static_cast<ALsizei>(sampleRate));

                alSourceQueueBuffers(sourceId, 1, &bufferIds[nextBuffer]);

                if (checkOpenALError())
                    throw SystemError("Failed to queue OpenAL buffer");

                ALint state;
                alGetSourcei(sourceId, AL_SOURCE_STATE, &state);
                if (state != AL_PLAYING)
                {
                    alSourcePlay(sourceId);

                    if (checkOpenALError())
                        throw SystemError("Failed to play OpenAL source");
                }

                // swap the buffer
                nextBuffer = (nextBuffer + 1) % 2;
            }
        }

        void AudioDeviceAL::run()
        {
#if OUZEL_MULTITHREADED
            while (running) process();
#endif
        }
    } // namespace audio
} // namespace ouzel

#endif
<commit_msg>Don't log errors in the destructor<commit_after>// Copyright (C) 2018 Elviss Strazdins
// This file is part of the Ouzel engine.

#include "core/Setup.h"

#if OUZEL_COMPILE_OPENAL

#if OUZEL_PLATFORM_IOS || OUZEL_PLATFORM_TVOS
#include <objc/message.h>
extern "C" id const AVAudioSessionCategoryAmbient;
#endif

#include "AudioDeviceAL.hpp"
#include "core/Engine.hpp"
#include "utils/Errors.hpp"

namespace ouzel
{
    namespace audio
    {
        AudioDeviceAL::AudioDeviceAL():
            AudioDevice(Audio::Driver::OPENAL)
        {
#if OUZEL_PLATFORM_IOS || OUZEL_PLATFORM_TVOS
            id audioSession = reinterpret_cast<id (*)(Class, SEL)>(&objc_msgSend)(objc_getClass("AVAudioSession"), sel_getUid("sharedInstance"));
            reinterpret_cast<BOOL (*)(id, SEL, id, id)>(&objc_msgSend)(audioSession, sel_getUid("setCategory:error:"), AVAudioSessionCategoryAmbient, nil);
#endif

#if OUZEL_MULTITHREADED
            running = false;
#endif

            std::fill(std::begin(bufferIds), std::end(bufferIds), 0);
        }

        AudioDeviceAL::~AudioDeviceAL()
        {
#if OUZEL_MULTITHREADED
            running = false;
            if (audioThread.isJoinable()) audioThread.join();
#endif

            if (sourceId)
            {
                alSourceStop(sourceId);
                alSourcei(sourceId, AL_BUFFER, 0);
                alDeleteSources(1, &sourceId);

                checkOpenALError();
            }

            for (ALuint bufferId : bufferIds)
            {
                if (bufferId)
                {
                    alDeleteBuffers(1, &bufferId);

                    checkOpenALError();
                }
            }

            if (context)
            {
                alcMakeContextCurrent(nullptr);
                alcDestroyContext(context);
            }

            if (device)
                alcCloseDevice(device);
        }

        void AudioDeviceAL::init(bool debugAudio)
        {
            AudioDevice::init(debugAudio);

            const ALCchar* deviceName = alcGetString(nullptr, ALC_DEFAULT_DEVICE_SPECIFIER);

            if (deviceName)
                Log(Log::Level::INFO) << "Using " << deviceName << " for audio";

            device = alcOpenDevice(deviceName);

            if (!device || checkALCError())
                throw SystemError("Failed to create OpenAL device");

            int capabilities[] =
            {
                ALC_FREQUENCY, 44100,
                ALC_STEREO_SOURCES, 4,
                0, 0
            };

            context = alcCreateContext(device, capabilities);

            if (checkALCError())
                throw SystemError("Failed to create OpenAL context");

            alcMakeContextCurrent(context);

            if (checkALCError())
                throw SystemError("Failed to make OpenAL context current");

#if !OUZEL_PLATFORM_EMSCRIPTEN
            format40 = alGetEnumValue("AL_FORMAT_QUAD16");
            format51 = alGetEnumValue("AL_FORMAT_51CHN16");
            format61 = alGetEnumValue("AL_FORMAT_61CHN16");
            format71 = alGetEnumValue("AL_FORMAT_71CHN16");
#endif

            if (checkOpenALError())
                Log(Log::Level::WARN) << "Failed to get OpenAL enum values";

            alGenSources(1, &sourceId);

            if (checkOpenALError())
                throw SystemError("Failed to create OpenAL source");

            alGenBuffers(2, bufferIds);

            if (checkOpenALError())
                throw SystemError("Failed to create OpenAL buffers");

            switch (channels)
            {
                case 1: format = AL_FORMAT_MONO16; break;
                case 2: format = AL_FORMAT_STEREO16; break;
                case 4: format = format40; break;
                case 6: format = format51; break;
                case 7: format = format61; break;
                case 8: format = format71; break;
                default:
                    throw SystemError("Invalid channel count");
            }

            sampleFormat = Audio::SampleFormat::SINT16;

            getData(bufferSize / (channels * sizeof(int16_t)), data);

            alBufferData(bufferIds[0], format,
                         data.data(),
                         static_cast<ALsizei>(data.size()),
                         static_cast<ALsizei>(sampleRate));

            getData(bufferSize / (channels * sizeof(int16_t)), data);

            alBufferData(bufferIds[1], format,
                         data.data(),
                         static_cast<ALsizei>(data.size()),
                         static_cast<ALsizei>(sampleRate));

            nextBuffer = 0;

            alSourceQueueBuffers(sourceId, 2, bufferIds);

            if (checkOpenALError())
                throw SystemError("Failed to queue OpenAL buffers");

            alSourcePlay(sourceId);

            if (checkOpenALError())
                throw SystemError("Failed to play OpenAL source");

#if OUZEL_MULTITHREADED
            running = true;
            audioThread = Thread(std::bind(&AudioDeviceAL::run, this), "Audio");
#endif
        }

        void AudioDeviceAL::process()
        {
            AudioDevice::process();

            alcMakeContextCurrent(context);

            if (checkALCError())
                throw SystemError("Failed to make OpenAL context current");

            ALint buffersProcessed;
            alGetSourcei(sourceId, AL_BUFFERS_PROCESSED, &buffersProcessed);

            if (checkOpenALError())
                throw SystemError("Failed to get processed buffer count");

            // requeue all processed buffers
            for (; buffersProcessed > 0; --buffersProcessed)
            {
                alSourceUnqueueBuffers(sourceId, 1, &bufferIds[nextBuffer]);

                if (checkOpenALError())
                    throw SystemError("Failed to unqueue OpenAL buffer");

                if (!getData(bufferSize / (channels * sizeof(int16_t)), data))
                    throw SystemError("Failed to get data");

                alBufferData(bufferIds[nextBuffer], format,
                             data.data(),
                             static_cast<ALsizei>(data.size()),
                             static_cast<ALsizei>(sampleRate));

                alSourceQueueBuffers(sourceId, 1, &bufferIds[nextBuffer]);

                if (checkOpenALError())
                    throw SystemError("Failed to queue OpenAL buffer");

                ALint state;
                alGetSourcei(sourceId, AL_SOURCE_STATE, &state);
                if (state != AL_PLAYING)
                {
                    alSourcePlay(sourceId);

                    if (checkOpenALError())
                        throw SystemError("Failed to play OpenAL source");
                }

                // swap the buffer
                nextBuffer = (nextBuffer + 1) % 2;
            }
        }

        void AudioDeviceAL::run()
        {
#if OUZEL_MULTITHREADED
            while (running) process();
#endif
        }
    } // namespace audio
} // namespace ouzel

#endif
<|endoftext|>


<commit_before>#include "controller/rocket_angular_acceleration_controller.hpp"

#include <algorithm>

#include "unit_config.hpp"

RocketAngularAccelerationController::RocketAngularAccelerationController()
  : rollAccPid(unit_config::ANGACC_X_KP, unit_config::ANGACC_X_KI, unit_config::ANGACC_X_KD),
    pitchAccPid(unit_config::ANGACC_Y_KP, unit_config::ANGACC_Y_KI, unit_config::ANGACC_Y_KD),
    yawAccPid(unit_config::ANGACC_Z_KP, unit_config::ANGACC_Z_KI, unit_config::ANGACC_Z_KD) {
}

actuator_setpoint_t RocketAngularAccelerationController::run(const attitude_estimate_t& estimate, const angular_acceleration_setpoint_t& input) {
  // Limit to maximum angular accelerations
  float rollAccSp = std::max(-unit_config::MAX_PITCH_ROLL_ACC, std::min(unit_config::MAX_PITCH_ROLL_ACC, input.roll_acc_sp));
  float pitchAccSp = std::max(-unit_config::MAX_PITCH_ROLL_ACC, std::min(unit_config::MAX_PITCH_ROLL_ACC, input.pitch_acc_sp));

  // Constants
  const float M_PI = 3.1415926535;
  const float F_LE   = M_PI * 7/18;    // Fin leading edge angle (rad)
  const float F_TR   = 0;              // Fin trailing edge distance past fin base (m)
  const float F_LEN  = 0.123;          // Fin length (m)
  const float F_AREA = 0.0027552;      // Fin area (m^2)
  const float F_CP   = 0.33 * F_LEN;   // Fin Cp (m)
  const float F_NUM  = 2;              // Number of fins
  const float F_D    = 0.06477;        // Distance from roll axis to fin Cp (m)
  const float I      = 1;              // TODO: Rocket rotational inertia

  // Sensor inputs
  float alt = 4900;   // Altitude (m)
  float v_rocket = 0;   // TODO: Magically figure this out from sensors (m/s)
  // TODO: Expand estimate to full 12-space

  // Calculate atmospheric pressure. We should eventually get this from MS5611.
  // Eq. 9 from "A Quick Derivation relating altitude to air pressure", PSAS,
  // 2004.
  float pressure = 100*pow((44331.514-alt)/11880.516, 5.255877);   // Pressure (Pa)
  float temp = 273.15;   // Absolute temperature (K)   TODO: add external thermistor

  // Calculate air density, speed of sound
  float p_air = pressure/(287.058*temp);   // Air density from ideal gas law (kg/m^3)
  float v_air = 20.05*sqrt(temp+273.15);   // Speed of sound (m/s)

  // Mach number
  float M = v_rocket / v_air;   // Mach number

  // Mach angle
  float u = asin(1/M);                   // Unicorn
  float beta = sqrt(pow(M,2) - 1);       // Bollocks
  float m = beta * tan(M_PI/2 - F_LE);   // Magic cot(x) = tan(pi/2 - x)

  // Elliptic Integral
  const float a[] = {0.44325141463,
                     0.06260601220,
                     0.04757383546,
                     0.01736506451};
  const float b[] = {0.24998368310,
                     0.09200180037,
                     0.04069697526,
                     0.00526449639};
  float E = 1;
  for (int i=0; i<4; i++) {
    E += a[i]*pow(m,2*(i+1)) + log(1/pow(m,2))*b[i]*pow(m,2*(i+1));   // 0 <= m < 1
  }

  // Lift coefficient
  float C_L = (2 * M_PI * m) / (E * beta);

  // Roll controller
  float torque = I * rollAccSp;
  float F_L = torque / (F_NUM * F_D);   // Force required per fin (N)

  // Fin controller
  float rollActuatorSp = (2*F_L) / (C_L * p_air * pow(v_air,2) * F_AREA);   // Output fin angle

  // Output
  actuator_setpoint_t setpoint {
    .roll_sp = rollActuatorSp,
    .pitch_sp = 0.0f,
    .yaw_sp = 0.0f,
    .throttle_sp = input.throttle_sp
  };

  return setpoint;
}
<commit_msg>Fix rocket equations for subsonic flight.<commit_after>#include "controller/rocket_angular_acceleration_controller.hpp"

#include <algorithm>

#include "unit_config.hpp"

RocketAngularAccelerationController::RocketAngularAccelerationController()
  : rollAccPid(unit_config::ANGACC_X_KP, unit_config::ANGACC_X_KI, unit_config::ANGACC_X_KD),
    pitchAccPid(unit_config::ANGACC_Y_KP, unit_config::ANGACC_Y_KI, unit_config::ANGACC_Y_KD),
    yawAccPid(unit_config::ANGACC_Z_KP, unit_config::ANGACC_Z_KI, unit_config::ANGACC_Z_KD) {
}

actuator_setpoint_t RocketAngularAccelerationController::run(const attitude_estimate_t& estimate, const angular_acceleration_setpoint_t& input) {
  // Limit to maximum angular accelerations
  float rollAccSp = std::max(-unit_config::MAX_PITCH_ROLL_ACC, std::min(unit_config::MAX_PITCH_ROLL_ACC, input.roll_acc_sp));
  float pitchAccSp = std::max(-unit_config::MAX_PITCH_ROLL_ACC, std::min(unit_config::MAX_PITCH_ROLL_ACC, input.pitch_acc_sp));

  // Constants
  const float M_PI = 3.1415926535;
  const float F_LE   = M_PI * 7/18;    // Fin leading edge angle (rad)
  const float F_TR   = 0;              // Fin trailing edge distance past fin base (m)
  const float F_LEN  = 0.123;          // Fin length (m)
  const float F_AREA = 0.0027552;      // Fin area (m^2)
  const float F_CP   = 0.33 * F_LEN;   // Fin Cp (m)
  const float F_NUM  = 2;              // Number of fins
  const float F_D    = 0.06477;        // Distance from roll axis to fin Cp (m)
  const float I      = 1;              // TODO: Rocket rotational inertia

  // Sensor inputs
  float alt = 1414;   // Altitude (m)
  float v_rocket = 50;   // TODO: Magically figure this out from sensors (m/s)
  // TODO: Expand estimate to full 12-space

  // Calculate atmospheric pressure. We should eventually get this from MS5611.
  // Eq. 9 from "A Quick Derivation relating altitude to air pressure", PSAS,
  // 2004.
  float pressure = 100*pow((44331.514-alt)/11880.516, 5.255877);   // Pressure (Pa)
  float temp = 273.15;   // Absolute temperature (K)   TODO: add external thermistor

  // Calculate air density, speed of sound
  float p_air = pressure/(287.058*temp);   // Air density from ideal gas law (kg/m^3)
  float v_air = 20.05*sqrt(temp);   // Speed of sound (m/s)

  // Mach number
  float M = v_rocket / v_air;   // Mach number

  // Mach angle
  //float u = asin(1/M);                   // Unicorn
  //float beta = sqrt(pow(M,2) - 1);       // Bollocks
  //float m = beta * tan(M_PI/2 - F_LE);   // Magic cot(x) = tan(pi/2 - x)

  // Elliptic Integral
  //const float a[] = {0.44325141463,
  //                   0.06260601220,
  //                   0.04757383546,
  //                   0.01736506451};
  //const float b[] = {0.24998368310,
  //                   0.09200180037,
  //                   0.04069697526,
  //                   0.00526449639};
  //float E = 1;
  //for (int i=0; i<4; i++) {
  //  E += a[i]*pow(m,2*(i+1)) + log(1/pow(m,2))*b[i]*pow(m,2*(i+1));   // 0 <= m < 1
  //}

  // Roll controller
  float torque = I * rollAccSp;
  float F_L = torque / (F_NUM * F_D);   // Force required per fin (N)

  // Lift coefficient
  float C_L = 2*F_L/(p_air * pow(v_rocket,2) * F_AREA);   // Subsonic
  //float C_L = (2 * M_PI * m) / (E * beta);   // Supersonic

  // Fin controller
  float rollActuatorSp = 0.5 + C_L/(2*M_PI);   // Fin angle

  // Output
  actuator_setpoint_t setpoint {
    .roll_sp = rollActuatorSp,
    .pitch_sp = 0.0f,
    .yaw_sp = 0.0f,
    .throttle_sp = input.throttle_sp
  };

  return setpoint;
}
<|endoftext|>


<commit_before>/** \brief Utility for creating ngram language models.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2019 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <fstream>
#include <stdexcept>
#include <cstdlib>
#include "BinaryIO.h"
#include "FileUtil.h"
#include "NGram.h"
#include "util.h"

int Main(int argc, char *argv[]) {
    if (argc != 3)
        ::Usage("language_blob language_model\n"
                "The \"language_blob\" should be a file containing example text w/o markup in whatever language.\n"
                "\"language_model\" should be named after the language followed by \".lm\".\n");

    std::ifstream input(argv[1]);
    if (not input)
        LOG_ERROR("failed to open \"" + std::string(argv[1]) + "\" for reading!");

    NGram::NGramCounts ngram_counts;
    NGram::SortedNGramCounts sorted_ngram_counts;
    NGram::CreateLanguageModel(input, &ngram_counts, &sorted_ngram_counts);

    const auto output(FileUtil::OpenOutputFileOrDie(argv[2]));
    BinaryIO::WriteOrDie(*output, ngram_counts.size());
    for (const auto &ngram_and_rank : sorted_ngram_counts) {
        BinaryIO::WriteOrDie(*output, ngram_and_rank.first);
        BinaryIO::WriteOrDie(*output, ngram_and_rank.second);
    }

    return EXIT_SUCCESS;
}
<commit_msg>Fixed a bug.<commit_after>/** \brief Utility for creating ngram language models.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2019 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <fstream>
#include <stdexcept>
#include <cstdlib>
#include "BinaryIO.h"
#include "FileUtil.h"
#include "NGram.h"
#include "util.h"

int Main(int argc, char *argv[]) {
    if (argc != 3)
        ::Usage("language_blob language_model\n"
                "The \"language_blob\" should be a file containing example text w/o markup in whatever language.\n"
                "\"language_model\" should be named after the language followed by \".lm\".\n");

    std::ifstream input(argv[1]);
    if (not input)
        LOG_ERROR("failed to open \"" + std::string(argv[1]) + "\" for reading!");

    NGram::NGramCounts ngram_counts;
    NGram::SortedNGramCounts sorted_ngram_counts;
    NGram::CreateLanguageModel(input, &ngram_counts, &sorted_ngram_counts);

    const auto output(FileUtil::OpenOutputFileOrDie(argv[2]));
    BinaryIO::WriteOrDie(*output, sorted_ngram_counts.size());
    for (const auto &ngram_and_rank : sorted_ngram_counts) {
        BinaryIO::WriteOrDie(*output, ngram_and_rank.first);
        BinaryIO::WriteOrDie(*output, ngram_and_rank.second);
    }

    return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>/** \brief MARC grep the Next Generation.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2019 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <cstdlib>
#include "Compiler.h"
#include "MARC.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "util.h"

namespace {

[[noreturn]] void Usage() {
    ::Usage("--query=query [--output-format=output_format] marc_file1 [marc_file2 .. marc_fileN]\n"
            "Queries have the following syntax:\n"
            "expression → term {OR term}\n"
            "term       → factor {AND factor}\n"
            "factor     → field_or_subfield_reference (== | !=) string_constant_or_regex\n"
            "factor     → NOT factor\n"
            "factor     → '(' expression ')'\n"
            );
    std::exit(EXIT_FAILURE);
}

enum TokenType { AND, OR, NOT, STRING_CONST, FUNC_CALL, OPEN_PAREN, CLOSE_PAREN, REGEX, EQUALS, NOT_EQUALS,
                 COMMA, ERROR, END_OF_QUERY };

class Tokenizer {
    struct FunctionDesc {
        std::string name_;
        unsigned argument_count_;
    };

    std::string::const_iterator next_ch_;
    const std::string::const_iterator end_;
    bool pushed_back_;
    TokenType last_token_;
    std::string last_error_message_;
    std::string last_string_;
    std::string last_function_name_;
    const static std::vector<FunctionDesc> function_descriptions_;
public:
    explicit Tokenizer(const std:: string &query): next_ch_(query.cbegin()), end_(query.cend()), pushed_back_(false) { }
    TokenType getNextToken();
    void ungetLastToken();
    inline const std::string &getLastErrorMessage() const { return last_error_message_; }
    inline const std::string &getLastString() const { return last_string_; }
    inline const std::string &getLastFunctionName() const { return last_function_name_; }
    static std::string TokenTypeToString(const TokenType token);
private:
    TokenType parseStringConstantOrRegex();
    bool isKnownFunctionName(const std::string &name_candidate) const;
};

const std::vector<Tokenizer::FunctionDesc> Tokenizer::function_descriptions_{ { "HasField", 1 }, { "HasSubfield", 2} };

TokenType Tokenizer::getNextToken() {
    if (pushed_back_) {
        pushed_back_ = false;
        return last_token_;
    }

    // Skip over spaces:
    while (next_ch_ != end_ and *next_ch_ == ' ')
        ++next_ch_;

    if (next_ch_ == end_)
        return last_token_ = END_OF_QUERY;

    if (*next_ch_ == '"' or *next_ch_ == '/')
        return last_token_ = parseStringConstantOrRegex();

    if (*next_ch_ == ',') {
        ++next_ch_;
        return last_token_ = COMMA;
    }

    if (*next_ch_ == '(') {
        ++next_ch_;
        return last_token_ = OPEN_PAREN;
    }

    if (*next_ch_ == ')') {
        ++next_ch_;
        return last_token_ = CLOSE_PAREN;
    }

    if (*next_ch_ == '=') {
        ++next_ch_;
        if (next_ch_ == end_ or *next_ch_ != '=') {
            last_error_message_ = "unexpected single equal sign found!";
            return ERROR;
        }
        return last_token_ = EQUALS;
    }

    if (*next_ch_ == '!') {
        ++next_ch_;
        if (next_ch_ == end_ or *next_ch_ != '=') {
            last_error_message_ = "unexpected single exclamation point found!";
            return ERROR;
        }
        return last_token_ = NOT_EQUALS;
    }

    if (unlikely(not StringUtil::IsAsciiLetter(*next_ch_))) {
        last_error_message_ = "expected ASCII letter!";
        return ERROR;
    }

    std::string id;
    while (next_ch_ != end_ and StringUtil::IsAsciiLetter(*next_ch_))
        id += *next_ch_;

    if (id == "AND")
        return last_token_ = AND;
    if (id == "OR")
        return last_token_ = OR;
    if (id == "NOT")
        return last_token_ = NOT;

    if (isKnownFunctionName(id)) {
        last_function_name_ = id;
        return last_token_ = FUNC_CALL;
    }

    last_error_message_ = "unknown function name \"" + id + "\"!";

    return ERROR;
}

void Tokenizer::ungetLastToken() {
    if (unlikely(pushed_back_))
        LOG_ERROR("can't push back two tokens in a row!");
    pushed_back_ = true;
}

TokenType Tokenizer::parseStringConstantOrRegex() {
    const char terminator(*next_ch_++);

    last_string_.clear();
    bool escaped(false);
    for (/* Intentionally empty! */; next_ch_ != end_; ++next_ch_) {
        if (escaped) {
            escaped = false;
            last_string_ += *next_ch_;
        } else if (*next_ch_ == terminator) {
            ++next_ch_;
            return (terminator == '"') ? STRING_CONST : REGEX;
        } else if (*next_ch_ == '\\')
            escaped = true;
        else
            last_string_ += *next_ch_;
    }

    last_error_message_ = "unterminated string constant or regex!";
    return ERROR;
}

bool Tokenizer::isKnownFunctionName(const std::string &name_candidate) const {
    for (const auto &function_description : function_descriptions_) {
        if (function_description.name_ == name_candidate)
            return true;
    }

    return false;
}

std::string Tokenizer::TokenTypeToString(const TokenType token) {
    switch (token) {
    case AND:
        return "AND";
    case OR:
        return "OR";
    case NOT:
        return "NOT";
    case STRING_CONST:
        return "string constant";
    case FUNC_CALL:
        return "function call";
    case OPEN_PAREN:
        return "(";
    case CLOSE_PAREN:
        return ")";
    case REGEX:
        return "regular expression";
    case EQUALS:
        return "==";
    case NOT_EQUALS:
        return "!=";
    case COMMA:
        return ",";
    case ERROR:
        return "unexpected input";
    case END_OF_QUERY:
        return "end-of-query";
    }
}

class Query {
    enum NodeType { AND_NODE, OR_NODE, STRING_COMPARISON_NODE, REGEX_COMPARISON_NODE };
    class Node {
    protected:
        Node() {}
    public:
        virtual ~Node() = 0;
        virtual NodeType getNodeType() const = 0;
        virtual bool eval(const MARC::Record &record) const = 0;
    };

    class AndNode final : public Node {
        std::vector<Node *> children_;
    public:
        AndNode() { }
        ~AndNode() {
            for (const auto child_node : children_)
                delete child_node;
        }
        virtual NodeType getNodeType() const override { return AND_NODE; }
        virtual bool eval(const MARC::Record &record) const override;
    };

    class OrNode final : public Node {
        std::vector<Node *> children_;
    public:
        OrNode() { }
        ~OrNode() {
            for (const auto child_node : children_)
                delete child_node;
        }
        virtual NodeType getNodeType() const override { return OR_NODE; }
        virtual bool eval(const MARC::Record &record) const override;
    };

    class StringComparisonNode final : public Node {
        MARC::Tag field_tag_;
        char subfield_code_;
        std::string string_const_;
        bool invert_;
    public:
        StringComparisonNode(const std::string &field_or_subfield_reference, const std::string &string_const, const bool invert)
            : field_tag_(field_or_subfield_reference.substr(0, MARC::Record::TAG_LENGTH)),
              subfield_code_(field_or_subfield_reference.length() > MARC::Record::TAG_LENGTH
                             ? field_or_subfield_reference[MARC::Record::TAG_LENGTH] : '\0'),
              string_const_(string_const), invert_(invert) { }
        ~StringComparisonNode() = default;
        virtual NodeType getNodeType() const override { return STRING_COMPARISON_NODE; }
        virtual bool eval(const MARC::Record &record) const override;
    };

    Tokenizer tokenizer_;
public:
    explicit Query(const std:: string &query);
    bool matched(const MARC::Record &record) const;
private:
    void ParseExpression();
    void ParseTerm();
    void ParseFactor();
};

Query::Node::~Node() {
    /* Intentionally empty! */
}

bool Query::AndNode::eval(const MARC::Record &record) const {
    for (const auto child_node : children_) {
        if (not child_node->eval(record))
            return false;
    }

    return true;
}

bool Query::OrNode::eval(const MARC::Record &record) const {
    for (const auto child_node : children_) {
        if (child_node->eval(record))
            return true;
    }

    return false;
}

bool Query::StringComparisonNode::eval(const MARC::Record &record) const {
    for (const auto &field : record.getTagRange(field_tag_)) {
        if (subfield_code_ == '\0') {
            if (field .getContents() == string_const_) {
                if (not invert_)
                    return true;
            } else if (invert_)
                return true;
        } else {
            const MARC::Subfields subfields(field.getSubfields());
            for (const auto &value_and_code : subfields) {
                if (value_and_code.code_ == subfield_code_) {
                    if (value_and_code.value_ == string_const_) {
                        if (not invert_)
                            return true;
                    } else if (invert_)
                        return true;
                }
            }
        }
    }

    return invert_;
}

Query::Query(const std:: string &query): tokenizer_(query) {
    ParseExpression();
}

bool Query::matched(const MARC::Record &/*record*/) const {
    return false;
}

void Query::ParseExpression() {
    ParseTerm();

    TokenType token(tokenizer_.getNextToken());
    while (token == OR) {
        ParseTerm();
        token = tokenizer_.getNextToken();
    }

    tokenizer_.ungetLastToken();
}

void Query::ParseTerm() {
    ParseFactor();

    TokenType token(tokenizer_.getNextToken());
    while (token == AND) {
        ParseFactor();
        token = tokenizer_.getNextToken();
    }

    tokenizer_.ungetLastToken();
}

void Query::ParseFactor() {
    TokenType token(tokenizer_.getNextToken());
    if (token == STRING_CONST) {
        if (tokenizer_.getLastString().length() != MARC::Record::TAG_LENGTH
            and tokenizer_.getLastString().length() != MARC::Record::TAG_LENGTH + 1)
            throw std::runtime_error("invalid field or subfield reference \"" + tokenizer_.getLastString() + "\"!");

        token = tokenizer_.getNextToken();
        if (token != EQUALS and token != NOT_EQUALS)
            throw std::runtime_error("expected == or != after field or subfield reference, found "
                                     + Tokenizer::TokenTypeToString(token) + " instead!");

        const TokenType last_token(token);
        token = tokenizer_.getNextToken();
        if (token != STRING_CONST and token != REGEX)
            throw std::runtime_error("expected a string constant or a regex after " + Tokenizer::TokenTypeToString(last_token) + ", found "
                                     + Tokenizer::TokenTypeToString(token) + " instead!");

        return;
    }

    if (token == NOT) {
        ParseFactor();
        return;
    }

    if (token != OPEN_PAREN)
        throw std::runtime_error("opening parenthesis, NOT or string constant expected found " + Tokenizer::TokenTypeToString(token)
                                 + "instead!");

    ParseExpression();

    token = tokenizer_.getNextToken();
    if (token != CLOSE_PAREN)
        throw std::runtime_error("closing parenthesis after expression expected, found " + Tokenizer::TokenTypeToString(token) + "instead!");
}

void ProcessRecords(const Query &query, MARC::Reader * const marc_reader) {
    unsigned record_count(0), matched_count(0);

    while (const MARC::Record record = marc_reader->read()) {
        ++record_count;

        if (not query.matched(record))
            continue;

        ++matched_count;
    }

    std::cerr << "Matched " << matched_count << " of " << record_count << " records.\n";
}

} // unnamed namespace

int Main(int argc, char *argv[]) {
    if (argc < 3)
        Usage();

    if (not StringUtil::StartsWith(argv[1], "--query="))
        LOG_ERROR("missing --query=...!");
    const std::string query_str(argv[1] + __builtin_strlen("--query="));
    Query query(query_str);

    if (argc < 3)
        Usage();

    for (int arg_no(1); arg_no < argc; ++arg_no) {
        auto marc_reader(MARC::Reader::Factory(argv[1]));
        ProcessRecords(query, marc_reader.get());
    }

    return EXIT_SUCCESS;
}
<commit_msg>Fixed a return value.<commit_after>/** \brief MARC grep the Next Generation.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2019 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <cstdlib>
#include "Compiler.h"
#include "MARC.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "util.h"

namespace {

[[noreturn]] void Usage() {
    ::Usage("--query=query [--output-format=output_format] marc_file1 [marc_file2 .. marc_fileN]\n"
            "Queries have the following syntax:\n"
            "expression → term {OR term}\n"
            "term       → factor {AND factor}\n"
            "factor     → field_or_subfield_reference (== | !=) string_constant_or_regex\n"
            "factor     → NOT factor\n"
            "factor     → '(' expression ')'\n"
            );
    std::exit(EXIT_FAILURE);
}

enum TokenType { AND, OR, NOT, STRING_CONST, FUNC_CALL, OPEN_PAREN, CLOSE_PAREN, REGEX, EQUALS, NOT_EQUALS,
                 COMMA, ERROR, END_OF_QUERY };

class Tokenizer {
    struct FunctionDesc {
        std::string name_;
        unsigned argument_count_;
    };

    std::string::const_iterator next_ch_;
    const std::string::const_iterator end_;
    bool pushed_back_;
    TokenType last_token_;
    std::string last_error_message_;
    std::string last_string_;
    std::string last_function_name_;
    const static std::vector<FunctionDesc> function_descriptions_;
public:
    explicit Tokenizer(const std:: string &query): next_ch_(query.cbegin()), end_(query.cend()), pushed_back_(false) { }
    TokenType getNextToken();
    void ungetLastToken();
    inline const std::string &getLastErrorMessage() const { return last_error_message_; }
    inline const std::string &getLastString() const { return last_string_; }
    inline const std::string &getLastFunctionName() const { return last_function_name_; }
    static std::string TokenTypeToString(const TokenType token);
private:
    TokenType parseStringConstantOrRegex();
    bool isKnownFunctionName(const std::string &name_candidate) const;
};

const std::vector<Tokenizer::FunctionDesc> Tokenizer::function_descriptions_{ { "HasField", 1 }, { "HasSubfield", 2} };

TokenType Tokenizer::getNextToken() {
    if (pushed_back_) {
        pushed_back_ = false;
        return last_token_;
    }

    // Skip over spaces:
    while (next_ch_ != end_ and *next_ch_ == ' ')
        ++next_ch_;

    if (next_ch_ == end_)
        return last_token_ = END_OF_QUERY;

    if (*next_ch_ == '"' or *next_ch_ == '/')
        return last_token_ = parseStringConstantOrRegex();

    if (*next_ch_ == ',') {
        ++next_ch_;
        return last_token_ = COMMA;
    }

    if (*next_ch_ == '(') {
        ++next_ch_;
        return last_token_ = OPEN_PAREN;
    }

    if (*next_ch_ == ')') {
        ++next_ch_;
        return last_token_ = CLOSE_PAREN;
    }

    if (*next_ch_ == '=') {
        ++next_ch_;
        if (next_ch_ == end_ or *next_ch_ != '=') {
            last_error_message_ = "unexpected single equal sign found!";
            return ERROR;
        }
        return last_token_ = EQUALS;
    }

    if (*next_ch_ == '!') {
        ++next_ch_;
        if (next_ch_ == end_ or *next_ch_ != '=') {
            last_error_message_ = "unexpected single exclamation point found!";
            return ERROR;
        }
        return last_token_ = NOT_EQUALS;
    }

    if (unlikely(not StringUtil::IsAsciiLetter(*next_ch_))) {
        last_error_message_ = "expected ASCII letter!";
        return ERROR;
    }

    std::string id;
    while (next_ch_ != end_ and StringUtil::IsAsciiLetter(*next_ch_))
        id += *next_ch_;

    if (id == "AND")
        return last_token_ = AND;
    if (id == "OR")
        return last_token_ = OR;
    if (id == "NOT")
        return last_token_ = NOT;

    if (isKnownFunctionName(id)) {
        last_function_name_ = id;
        return last_token_ = FUNC_CALL;
    }

    last_error_message_ = "unknown function name \"" + id + "\"!";

    return ERROR;
}

void Tokenizer::ungetLastToken() {
    if (unlikely(pushed_back_))
        LOG_ERROR("can't push back two tokens in a row!");
    pushed_back_ = true;
}

TokenType Tokenizer::parseStringConstantOrRegex() {
    const char terminator(*next_ch_++);

    last_string_.clear();
    bool escaped(false);
    for (/* Intentionally empty! */; next_ch_ != end_; ++next_ch_) {
        if (escaped) {
            escaped = false;
            last_string_ += *next_ch_;
        } else if (*next_ch_ == terminator) {
            ++next_ch_;
            return (terminator == '"') ? STRING_CONST : REGEX;
        } else if (*next_ch_ == '\\')
            escaped = true;
        else
            last_string_ += *next_ch_;
    }

    last_error_message_ = "unterminated string constant or regex!";
    return ERROR;
}

bool Tokenizer::isKnownFunctionName(const std::string &name_candidate) const {
    for (const auto &function_description : function_descriptions_) {
        if (function_description.name_ == name_candidate)
            return true;
    }

    return false;
}

std::string Tokenizer::TokenTypeToString(const TokenType token) {
    switch (token) {
    case AND:
        return "AND";
    case OR:
        return "OR";
    case NOT:
        return "NOT";
    case STRING_CONST:
        return "string constant";
    case FUNC_CALL:
        return "function call";
    case OPEN_PAREN:
        return "(";
    case CLOSE_PAREN:
        return ")";
    case REGEX:
        return "regular expression";
    case EQUALS:
        return "==";
    case NOT_EQUALS:
        return "!=";
    case COMMA:
        return ",";
    case ERROR:
        return "unexpected input";
    case END_OF_QUERY:
        return "end-of-query";
    }
}

class Query {
    enum NodeType { AND_NODE, OR_NODE, STRING_COMPARISON_NODE, REGEX_COMPARISON_NODE };
    class Node {
    protected:
        Node() {}
    public:
        virtual ~Node() = 0;
        virtual NodeType getNodeType() const = 0;
        virtual bool eval(const MARC::Record &record) const = 0;
    };

    class AndNode final : public Node {
        std::vector<Node *> children_;
    public:
        AndNode() { }
        ~AndNode() {
            for (const auto child_node : children_)
                delete child_node;
        }
        virtual NodeType getNodeType() const override { return AND_NODE; }
        virtual bool eval(const MARC::Record &record) const override;
    };

    class OrNode final : public Node {
        std::vector<Node *> children_;
    public:
        OrNode() { }
        ~OrNode() {
            for (const auto child_node : children_)
                delete child_node;
        }
        virtual NodeType getNodeType() const override { return OR_NODE; }
        virtual bool eval(const MARC::Record &record) const override;
    };

    class StringComparisonNode final : public Node {
        MARC::Tag field_tag_;
        char subfield_code_;
        std::string string_const_;
        bool invert_;
    public:
        StringComparisonNode(const std::string &field_or_subfield_reference, const std::string &string_const, const bool invert)
            : field_tag_(field_or_subfield_reference.substr(0, MARC::Record::TAG_LENGTH)),
              subfield_code_(field_or_subfield_reference.length() > MARC::Record::TAG_LENGTH
                             ? field_or_subfield_reference[MARC::Record::TAG_LENGTH] : '\0'),
              string_const_(string_const), invert_(invert) { }
        ~StringComparisonNode() = default;
        virtual NodeType getNodeType() const override { return STRING_COMPARISON_NODE; }
        virtual bool eval(const MARC::Record &record) const override;
    };

    Tokenizer tokenizer_;
public:
    explicit Query(const std:: string &query);
    bool matched(const MARC::Record &record) const;
private:
    void ParseExpression();
    void ParseTerm();
    void ParseFactor();
};

Query::Node::~Node() {
    /* Intentionally empty! */
}

bool Query::AndNode::eval(const MARC::Record &record) const {
    for (const auto child_node : children_) {
        if (not child_node->eval(record))
            return false;
    }

    return true;
}

bool Query::OrNode::eval(const MARC::Record &record) const {
    for (const auto child_node : children_) {
        if (child_node->eval(record))
            return true;
    }

    return false;
}

bool Query::StringComparisonNode::eval(const MARC::Record &record) const {
    for (const auto &field : record.getTagRange(field_tag_)) {
        if (subfield_code_ == '\0') {
            if (field .getContents() == string_const_) {
                if (not invert_)
                    return true;
            } else if (invert_)
                return true;
        } else {
            const MARC::Subfields subfields(field.getSubfields());
            for (const auto &value_and_code : subfields) {
                if (value_and_code.code_ == subfield_code_) {
                    if (value_and_code.value_ == string_const_) {
                        if (not invert_)
                            return true;
                    } else if (invert_)
                        return true;
                }
            }
        }
    }

    return false;
}

Query::Query(const std:: string &query): tokenizer_(query) {
    ParseExpression();
}

bool Query::matched(const MARC::Record &/*record*/) const {
    return false;
}

void Query::ParseExpression() {
    ParseTerm();

    TokenType token(tokenizer_.getNextToken());
    while (token == OR) {
        ParseTerm();
        token = tokenizer_.getNextToken();
    }

    tokenizer_.ungetLastToken();
}

void Query::ParseTerm() {
    ParseFactor();

    TokenType token(tokenizer_.getNextToken());
    while (token == AND) {
        ParseFactor();
        token = tokenizer_.getNextToken();
    }

    tokenizer_.ungetLastToken();
}

void Query::ParseFactor() {
    TokenType token(tokenizer_.getNextToken());
    if (token == STRING_CONST) {
        if (tokenizer_.getLastString().length() != MARC::Record::TAG_LENGTH
            and tokenizer_.getLastString().length() != MARC::Record::TAG_LENGTH + 1)
            throw std::runtime_error("invalid field or subfield reference \"" + tokenizer_.getLastString() + "\"!");

        token = tokenizer_.getNextToken();
        if (token != EQUALS and token != NOT_EQUALS)
            throw std::runtime_error("expected == or != after field or subfield reference, found "
                                     + Tokenizer::TokenTypeToString(token) + " instead!");

        const TokenType last_token(token);
        token = tokenizer_.getNextToken();
        if (token != STRING_CONST and token != REGEX)
            throw std::runtime_error("expected a string constant or a regex after " + Tokenizer::TokenTypeToString(last_token) + ", found "
                                     + Tokenizer::TokenTypeToString(token) + " instead!");

        return;
    }

    if (token == NOT) {
        ParseFactor();
        return;
    }

    if (token != OPEN_PAREN)
        throw std::runtime_error("opening parenthesis, NOT or string constant expected found " + Tokenizer::TokenTypeToString(token)
                                 + "instead!");

    ParseExpression();

    token = tokenizer_.getNextToken();
    if (token != CLOSE_PAREN)
        throw std::runtime_error("closing parenthesis after expression expected, found " + Tokenizer::TokenTypeToString(token) + "instead!");
}

void ProcessRecords(const Query &query, MARC::Reader * const marc_reader) {
    unsigned record_count(0), matched_count(0);

    while (const MARC::Record record = marc_reader->read()) {
        ++record_count;

        if (not query.matched(record))
            continue;

        ++matched_count;
    }

    std::cerr << "Matched " << matched_count << " of " << record_count << " records.\n";
}

} // unnamed namespace

int Main(int argc, char *argv[]) {
    if (argc < 3)
        Usage();

    if (not StringUtil::StartsWith(argv[1], "--query="))
        LOG_ERROR("missing --query=...!");
    const std::string query_str(argv[1] + __builtin_strlen("--query="));
    Query query(query_str);

    if (argc < 3)
        Usage();

    for (int arg_no(1); arg_no < argc; ++arg_no) {
        auto marc_reader(MARC::Reader::Factory(argv[1]));
        ProcessRecords(query, marc_reader.get());
    }

    return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>#include <iostream>

class Hi {
public:
    Hi() {
        std::cout << "Hi constructed" << std::endl;
    }

    ~Hi() {
        std::cout << "Hi destructed" << std::endl;
    }
};

int main() {
    Hi *my = new Hi();

    return 0;
}
<commit_msg>[trash] refs/new/constructor/destructor<commit_after>#include <iostream>

class Hi {
public:
    int value;

    Hi(int value) : value(value) {
        std::cout << "Hi(" << this->value << ") constructed" << std::endl;
    }

    Hi(const Hi &other) : value(other.value) {
        std::cout << "Hi(" << this->value << ") constructed from Hi(" << other.value << ")" << std::endl;
    }

    ~Hi() {
        std::cout << "Hi(" << value << ") destructed" << std::endl;
    }

    Hi &operator=(const Hi &rhs) {
        std::cout << "Hi(" << this->value << ") := Hi(" << rhs.value << ")" << std::endl;

        if (this != &rhs) {
            value = rhs.value;
        }

        return *this;
    }
};

int main() {
    Hi &a0 = *new Hi(0);
    Hi *a1 = new Hi(1);

    return 0;
}
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2016 Northeastern University
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

#include "include/cpu_operations.h"
#include <unistd.h>
#include <stdexcept>
#include <iostream>
#include "Eigen/Dense"
#include "include/matrix.h"
#include "include/vector.h"

namespace Nice {

// This function returns the transpose of a matrix
template<typename T>
Matrix<T> CpuOperations<T>::Transpose(const Matrix<T> &a) {
  return a.transpose();  // Return transpose
}

template<typename T>
Vector<T> CpuOperations<T>::Transpose(const Vector<T> &a) {
  return a.transpose();
}

// Returns the resulting matrix that is created by running a logical or
// operation on the two input matrices
template<typename T>
Matrix<bool> CpuOperations<T>::LogicalOr(const Matrix<bool> &a,
                                        const Matrix<bool> &b) {
  if ((a.rows() != b.rows()) || (a.cols() != b.cols())) {
    std::cout << std::endl << "ERROR: MATRICES ARE NOT THE SAME SIZE!"
    << std::endl << std::endl;
      exit(-1);  // Exits the program
  } else if (b.rows() == 0 || b.cols() == 0 || a.rows() == 0 || a.cols() == 0) {
    std::cout << std::endl << "ERROR: EMPTY MATRIX AS ARGUMENT!"
    << std::endl << std::endl;
    exit(-1);  // Exits the program
  }
  return (a.array() || b.array());
}

// Returns the resulting vector that is created by running a logical or
// operation on the two input vectors
template<typename T>
Vector<bool> CpuOperations<T>::LogicalOr(const Vector<bool> &a,
                                        const Vector<bool> &b) {
  if ( a.size() != b.size() ) {
    std::cout << std::endl << "ERROR: VECTORS ARE NOT THE SAME SIZE!"
    << std::endl << std::endl;
    exit(-1);  // Exits the program
  } else if (a.size() == 0 || b.size() == 0) {
    std::cout << std::endl << "ERROR: EMPTY VECTOR AS ARGUMENT!"
    << std::endl << std::endl;
    exit(-1);  // Exits the program
  }
  return (a.array() || b.array());
}

template<typename T>
Matrix<bool> CpuOperations<T>::LogicalNot(const Matrix<bool> &a) {
  Matrix<bool> b = a.replicate(1, 1);
  int r;
  // Iterate through the copied matrix
  for (r = 0; r < b.rows(); ++r) {
    for (int c = 0; c < b.cols(); ++c) {
      b(r, c) = !b(r, c);
    }
  }
  if (b.rows() == 0 || b.cols() == 0) {
    std::cout << std::endl << "ERROR: EMPTY MATRIX AS ARGUMENT!"
    << std::endl << std::endl;
    exit(-1);  // Exits the program
  }
  return b;
}

template<typename T>
Vector<bool> CpuOperations<T>::LogicalNot(const Vector<bool> &a) {
  Vector<bool> b = a.replicate(1, 1);
  int i;
  // Iterate through vector
  for (i = 0; i < b.size(); ++i) {
    b(i) = !b(i);
  }
  if (a.size() == 0) {
    std::cout << std::endl << "ERROR: EMPTY VECTOR AS ARGUMENT!"
    << std::endl << std::endl;
    exit(-1);  // Exits the program
    }
  return b;
}

// Scalar-matrix multiplication
template<typename T>
Matrix<T> CpuOperations<T>::Multiply(const Matrix<T> &a, const T &scalar) {
    return scalar * a;
}

// Matrix-matrix multiplication
template<typename T>
Matrix<T> CpuOperations<T>::Multiply(const Matrix<T> &a, const Matrix<T> &b) {
    return a * b;
}

// Trace of a matrix
template<typename T>
T CpuOperations<T>::Trace(const Matrix<T> &a) {
    return a.trace();
}

/*
// Rank of a matrix
template<typename T>
T CpuOperations<T>::Rank(const Matrix<T> &a) {
    return a.rank();
}
*/

// This function returns the logical AND of two boolean matrices
template<typename T>
Matrix<bool> CpuOperations<T>::LogicalAnd(const Matrix<bool> &a,
                                          const Matrix<bool> &b) {
  // Checks to see that the number of rows and columns are the same
  if ((a.rows() != b.rows()) || (a.cols() != b.cols())) {
    std::cout << "/nERROR: MATRICES ARE NOT THE SAME SIZE!/n/n";
    exit(1);  // Exits the program
  }
  return (a.array() && b.array());
  // Will return a matrix due to implicit conversion
}

// This function returns the outer product of he two passed in vectors
template<typename T>
Matrix<T> CpuOperations<T>::OuterProduct(const Vector<T> &a,
                                         const Vector<T> &b) {
  if (a.size() == 0 || b.size() == 0) {
    std::cout << std::endl << "ERROR: EMPTY VECTOR AS ARGUMENT!" << std::endl << std::endl;
    exit(-1);
  }
  return a * b.transpose();
}

template class CpuOperations<int>;
template class CpuOperations<float>;
template class CpuOperations<double>;
template class CpuOperations<bool>;

}  //  namespace Nice

<commit_msg>Adjusted for cerr and exit(1) format<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2016 Northeastern University
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

#include "include/cpu_operations.h"
#include <unistd.h>
#include <stdexcept>
#include <iostream>
#include "Eigen/Dense"
#include "include/matrix.h"
#include "include/vector.h"

namespace Nice {

// This function returns the transpose of a matrix
template<typename T>
Matrix<T> CpuOperations<T>::Transpose(const Matrix<T> &a) {
  return a.transpose();  // Return transpose
}

template<typename T>
Vector<T> CpuOperations<T>::Transpose(const Vector<T> &a) {
  return a.transpose();
}

// Returns the resulting matrix that is created by running a logical or
// operation on the two input matrices
template<typename T>
Matrix<bool> CpuOperations<T>::LogicalOr(const Matrix<bool> &a,
                                        const Matrix<bool> &b) {
  if ((a.rows() != b.rows()) || (a.cols() != b.cols())) {
    std::cerr << std::endl << "ERROR: MATRICES ARE NOT THE SAME SIZE!"
    << std::endl << std::endl;
      exit(1);  // Exits the program
  } else if (b.rows() == 0 || b.cols() == 0 || a.rows() == 0 || a.cols() == 0) {
    std::cerr << std::endl << "ERROR: EMPTY MATRIX AS ARGUMENT!"
    << std::endl << std::endl;
    exit(1);  // Exits the program
  }
  return (a.array() || b.array());
}

// Returns the resulting vector that is created by running a logical or
// operation on the two input vectors
template<typename T>
Vector<bool> CpuOperations<T>::LogicalOr(const Vector<bool> &a,
                                        const Vector<bool> &b) {
  if ( a.size() != b.size() ) {
    std::cerr << std::endl << "ERROR: VECTORS ARE NOT THE SAME SIZE!"
    << std::endl << std::endl;
    exit(1);  // Exits the program
  } else if (a.size() == 0 || b.size() == 0) {
    std::cerr << std::endl << "ERROR: EMPTY VECTOR AS ARGUMENT!"
    << std::endl << std::endl;
    exit(1);  // Exits the program
  }
  return (a.array() || b.array());
}

template<typename T>
Matrix<bool> CpuOperations<T>::LogicalNot(const Matrix<bool> &a) {
  Matrix<bool> b = a.replicate(1, 1);
  int r;
  // Iterate through the copied matrix
  for (r = 0; r < b.rows(); ++r) {
    for (int c = 0; c < b.cols(); ++c) {
      b(r, c) = !b(r, c);
    }
  }
  if (b.rows() == 0 || b.cols() == 0) {
    std::cerr << std::endl << "ERROR: EMPTY MATRIX AS ARGUMENT!"
    << std::endl << std::endl;
    exit(1);  // Exits the program
  }
  return b;
}

template<typename T>
Vector<bool> CpuOperations<T>::LogicalNot(const Vector<bool> &a) {
  Vector<bool> b = a.replicate(1, 1);
  int i;
  // Iterate through vector
  for (i = 0; i < b.size(); ++i) {
    b(i) = !b(i);
  }
  if (a.size() == 0) {
    std::cerr << std::endl << "ERROR: EMPTY VECTOR AS ARGUMENT!"
    << std::endl << std::endl;
    exit(1);  // Exits the program
    }
  return b;
}

// Scalar-matrix multiplication
template<typename T>
Matrix<T> CpuOperations<T>::Multiply(const Matrix<T> &a, const T &scalar) {
    return scalar * a;
}

// Matrix-matrix multiplication
template<typename T>
Matrix<T> CpuOperations<T>::Multiply(const Matrix<T> &a, const Matrix<T> &b) {
    return a * b;
}

// Trace of a matrix
template<typename T>
T CpuOperations<T>::Trace(const Matrix<T> &a) {
    return a.trace();
}

/*
// Rank of a matrix
template<typename T>
T CpuOperations<T>::Rank(const Matrix<T> &a) {
    return a.rank();
}
*/

// This function returns the logical AND of two boolean matrices
template<typename T>
Matrix<bool> CpuOperations<T>::LogicalAnd(const Matrix<bool> &a,
                                          const Matrix<bool> &b) {
  // Checks to see that the number of rows and columns are the same
  if ((a.rows() != b.rows()) || (a.cols() != b.cols())) {
    std::cerr << "/nERROR: MATRICES ARE NOT THE SAME SIZE!/n/n";
    exit(1);  // Exits the program
  }
  return (a.array() && b.array());
  // Will return a matrix due to implicit conversion
}

// This function returns the outer product of he two passed in vectors
template<typename T>
Matrix<T> CpuOperations<T>::OuterProduct(const Vector<T> &a,
                                         const Vector<T> &b) {
  if (a.size() == 0 || b.size() == 0) {
    std::cerr << std::endl << "ERROR: EMPTY VECTOR AS ARGUMENT!" << std::endl << std::endl;
    exit(1);
  }
  return a * b.transpose();
}

template class CpuOperations<int>;
template class CpuOperations<float>;
template class CpuOperations<double>;
template class CpuOperations<bool>;

}  //  namespace Nice

<|endoftext|>


<commit_before>#include"MatasanoConverter.h"

// styled according to the Google C++ Style guide
// https://google.github.io/styleguide/cppguide.html

uint8_t NumberFromHexChar(char hex_character) {
	if (48 <= hex_character && hex_character <= 57) { //0-9
		return (uint8_t) hex_character - 48;
	} else if (65 <= hex_character && hex_character <= 70) { //A-F
		return (uint8_t) hex_character - 55;
	} else if (97 <= hex_character && hex_character <= 102) { //a-f
		return (uint8_t) hex_character - 87;
	} else { //other - error!
		throw std::invalid_argument("numberFromHexChar: hex_character not valid.");
	}
}

std::vector<uint8_t> Base64Splitter(std::vector<uint8_t> eight_bit_number_array) {
	uint8_t temp;
	std::vector<uint8_t> base_64_output;
	base_64_output.clear(); //may be redundant

	for (int i = 0; i < eight_bit_number_array.size(); i+=3) { //three 8 bit numbers are taken at once and converted into four 6 bit numbers
		temp = eight_bit_number_array[i] >> 2;
		base_64_output.push_back(temp);
		temp = (eight_bit_number_array[ i ] & 0x3) << 4 | eight_bit_number_array[i+1] >> 4;
		base_64_output.push_back(temp);
		temp = (eight_bit_number_array[i+1] & 0xF) << 2 | eight_bit_number_array[i+2] >> 6;
		base_64_output.push_back(temp);
		temp = eight_bit_number_array[i+2] & 0x3F;
		base_64_output.push_back(temp);
	}

	return base_64_output;
}

std::vector<uint8_t> HexSplitter(std::vector<uint8_t> eight_bit_number_array) {
	uint8_t temp;
	std::vector<uint8_t> hex_output;
	hex_output.clear(); //may be redundant

	for (int i = 0; i < eight_bit_number_array.size(); i++) {
		temp = eight_bit_number_array[i] >> 4;
		hex_output.push_back(temp);
		temp = eight_bit_number_array[i] & 0xF;
		hex_output.push_back(temp);
	}

	return hex_output;
}

char Base64Character(uint8_t base_64_number) {
	if (0 <= base_64_number && base_64_number <= 25) { //upper case characters
		return (char) base_64_number + 65;
	} else if (26 <= base_64_number && base_64_number <= 51) { //lower case characters
		return (char) base_64_number + 71;
	} else if (52 <= base_64_number && base_64_number <= 61) { //numbers
		return (char) base_64_number - 4;
	} else if (62 == base_64_number) { //+
		return '+';
	} else if (63 == base_64_number) { // /
		return '/';
	} else {
		throw std::invalid_argument("Base64Character: invalid base64 number (not a number 0-63)");
	}
}

char HexCharacter(uint8_t hex_number) {
	if (0 <= hex_number && hex_number <= 9) {
		return (char) hex_number + 48;
	} else if (10 <= hex_number && hex_number <= 15) {
		return (char) hex_number + 87;
	} else {
		throw std::invalid_argument("HexCharacter: invalid hex number (not a number 0-15)");
	}
}

uint8_t CombineHex(uint8_t msb, uint8_t lsb) {
	return (msb << 4) | lsb;
}

std::string GetBase64String(std::vector<uint8_t> input_vector) {
	std::vector<uint8_t> data_copy = input_vector; //create copy that can be safely modified if needed
	char padding_bytes; //counter to record number of bytes added - value will be 0, 1 or 2
	std::string base_64_string = "";

	if (data_copy.size() % 3 == 1) {
		//add two pad bytes
		data_copy.push_back(0);
		data_copy.push_back(0);
		padding_bytes = 2;
	} else if (data_copy.size() % 3 == 2) {
		//add one pad byte
		data_copy.push_back(0);
		padding_bytes = 1;
	} else {
		//no padding required
		padding_bytes = 0;
	}

	data_copy = Base64Splitter(data_copy);

	for (int i = 0; i < data_copy.size(); i++) {
		base_64_string += Base64Character(data_copy[i]);
	}

	if (padding_bytes == 1) {
		base_64_string.replace(base_64_string.length() - 1, 1, "=", 0, 1);
	} else if (padding_bytes == 2) {
		base_64_string.replace(base_64_string.length() - 2, 2, "==", 0, 2);
	}

	data_copy.clear(); //destroy the vector as the copy is no longer needed - maybe redundant

	return base_64_string;
}

std::string GetHexString(std::vector<uint8_t> input_vector) {
	std::vector<uint8_t> data_copy = input_vector; //create copy that can be safely modified if needed
	std::string hex_string = "";

	data_copy = HexSplitter(data_copy);

	for (int i = 0; i < data_copy.size(); i++) {
		hex_string += HexCharacter(data_copy[i]);
	}

	return hex_string;
}

std::vector<uint8_t> HexStringConvert(std::string input_string) {
	//need to loop over each character and add two characters at a time to the uint8_t array
	std::vector<uint8_t> output_vector;
	uint32_t max = input_string.length();
	uint8_t a = 0, b = 0;

	output_vector.clear(); //may be redundant

	if (max % 2 != 0) {
		throw std::invalid_argument("HexStringInput: inputString not valid hex - uneven length.");
	}

	for(uint32_t i = 0; i < max; i+=2) {
		a = NumberFromHexChar(input_string[i]);
		b = NumberFromHexChar(input_string[i + 1]);
		output_vector.push_back( CombineHex( a , b ) );
	}

	return output_vector;
}

std::string StringFromVectorASCII(std::vector<uint8_t> input_vector) {
	std::string out = "";

	for (uint32_t i = 0; i < input_vector.size(); i++) {
		out += (char) input_vector[i];
	}

	return out;
}

std::string StringFromByteVector(std::vector<uint8_t> input_vector, std::string output_type) {
	std::string temp; //needed for upper case hex output
	if (output_type.compare("base64") == 0 || output_type.compare("b64") == 0 || output_type.compare("Base64") == 0 || output_type.compare("B64") == 0) {
		return GetBase64String(input_vector);
	} else if (output_type.compare("hex") == 0 || output_type.compare("h") == 0) { //returns lower case hex output
		return GetHexString(input_vector);
	} else if (output_type.compare("H") == 0 || output_type.compare("Hex") == 0 || output_type.compare("HEX") == 0) { //returns upper case hex output
		temp = GetHexString(input_vector);
		transform(temp.begin(), temp.end(), temp.begin(), ::toupper);
		return temp;
	} else if (output_type.compare("ASCII") == 0 || output_type.compare("ascii") == 0) {
		return StringFromVectorASCII(input_vector);
	} else {
		throw std::invalid_argument("GetStringOutput: unknown output_type.");
	}
}

std::vector<uint8_t> ByteVectorFromString(std::string input_string, std::string input_type) {
	//this function uses an if statement and the hexStringInput function to allow for easy expansion of the MatasanoConverter class
	//e.g. for base64 input
	if (input_type.compare("hex") == 0 || input_type.compare("Hex") == 0 || input_type.compare("h") == 0) {
		return HexStringConvert(input_string);
	} else {
		throw std::invalid_argument("inputString: unknown inputType.");
	}
}

<commit_msg>Add ASCII to byte values convertion<commit_after>#include"MatasanoConverter.h"

// styled according to the Google C++ Style guide
// https://google.github.io/styleguide/cppguide.html

uint8_t NumberFromHexChar(char hex_character) {
	if (48 <= hex_character && hex_character <= 57) { //0-9
		return (uint8_t) hex_character - 48;
	} else if (65 <= hex_character && hex_character <= 70) { //A-F
		return (uint8_t) hex_character - 55;
	} else if (97 <= hex_character && hex_character <= 102) { //a-f
		return (uint8_t) hex_character - 87;
	} else { //other - error!
		throw std::invalid_argument("numberFromHexChar: hex_character not valid.");
	}
}

std::vector<uint8_t> Base64Splitter(std::vector<uint8_t> eight_bit_number_array) {
	uint8_t temp;
	std::vector<uint8_t> base_64_output;
	base_64_output.clear(); //may be redundant

	for (int i = 0; i < eight_bit_number_array.size(); i+=3) { //three 8 bit numbers are taken at once and converted into four 6 bit numbers
		temp = eight_bit_number_array[i] >> 2;
		base_64_output.push_back(temp);
		temp = (eight_bit_number_array[ i ] & 0x3) << 4 | eight_bit_number_array[i+1] >> 4;
		base_64_output.push_back(temp);
		temp = (eight_bit_number_array[i+1] & 0xF) << 2 | eight_bit_number_array[i+2] >> 6;
		base_64_output.push_back(temp);
		temp = eight_bit_number_array[i+2] & 0x3F;
		base_64_output.push_back(temp);
	}

	return base_64_output;
}

std::vector<uint8_t> HexSplitter(std::vector<uint8_t> eight_bit_number_array) {
	uint8_t temp;
	std::vector<uint8_t> hex_output;
	hex_output.clear(); //may be redundant

	for (int i = 0; i < eight_bit_number_array.size(); i++) {
		temp = eight_bit_number_array[i] >> 4;
		hex_output.push_back(temp);
		temp = eight_bit_number_array[i] & 0xF;
		hex_output.push_back(temp);
	}

	return hex_output;
}

char Base64Character(uint8_t base_64_number) {
	if (0 <= base_64_number && base_64_number <= 25) { //upper case characters
		return (char) base_64_number + 65;
	} else if (26 <= base_64_number && base_64_number <= 51) { //lower case characters
		return (char) base_64_number + 71;
	} else if (52 <= base_64_number && base_64_number <= 61) { //numbers
		return (char) base_64_number - 4;
	} else if (62 == base_64_number) { //+
		return '+';
	} else if (63 == base_64_number) { // /
		return '/';
	} else {
		throw std::invalid_argument("Base64Character: invalid base64 number (not a number 0-63)");
	}
}

char HexCharacter(uint8_t hex_number) {
	if (0 <= hex_number && hex_number <= 9) {
		return (char) hex_number + 48;
	} else if (10 <= hex_number && hex_number <= 15) {
		return (char) hex_number + 87;
	} else {
		throw std::invalid_argument("HexCharacter: invalid hex number (not a number 0-15)");
	}
}

uint8_t CombineHex(uint8_t msb, uint8_t lsb) {
	return (msb << 4) | lsb;
}

std::string GetBase64String(std::vector<uint8_t> input_vector) {
	std::vector<uint8_t> data_copy = input_vector; //create copy that can be safely modified if needed
	char padding_bytes; //counter to record number of bytes added - value will be 0, 1 or 2
	std::string base_64_string = "";

	if (data_copy.size() % 3 == 1) {
		//add two pad bytes
		data_copy.push_back(0);
		data_copy.push_back(0);
		padding_bytes = 2;
	} else if (data_copy.size() % 3 == 2) {
		//add one pad byte
		data_copy.push_back(0);
		padding_bytes = 1;
	} else {
		//no padding required
		padding_bytes = 0;
	}

	data_copy = Base64Splitter(data_copy);

	for (int i = 0; i < data_copy.size(); i++) {
		base_64_string += Base64Character(data_copy[i]);
	}

	if (padding_bytes == 1) {
		base_64_string.replace(base_64_string.length() - 1, 1, "=", 0, 1);
	} else if (padding_bytes == 2) {
		base_64_string.replace(base_64_string.length() - 2, 2, "==", 0, 2);
	}

	data_copy.clear(); //destroy the vector as the copy is no longer needed - maybe redundant

	return base_64_string;
}

std::string GetHexString(std::vector<uint8_t> input_vector) {
	std::vector<uint8_t> data_copy = input_vector; //create copy that can be safely modified if needed
	std::string hex_string = "";

	data_copy = HexSplitter(data_copy);

	for (int i = 0; i < data_copy.size(); i++) {
		hex_string += HexCharacter(data_copy[i]);
	}

	return hex_string;
}

std::vector<uint8_t> HexStringConvert(std::string input_string) {
	//need to loop over each character and add two characters at a time to the uint8_t array
	std::vector<uint8_t> output_vector;
	uint32_t max = input_string.length();
	uint8_t a = 0, b = 0;

	output_vector.clear(); //may be redundant

	if (max % 2 != 0) {
		throw std::invalid_argument("HexStringInput: inputString not valid hex - uneven length.");
	}

	for(uint32_t i = 0; i < max; i+=2) {
		a = NumberFromHexChar(input_string[i]);
		b = NumberFromHexChar(input_string[i + 1]);
		output_vector.push_back( CombineHex( a , b ) );
	}

	return output_vector;
}

std::vector<uint8_t> ASCIIStringConvert(std::string input_string) {
	std::vector<uint8_t> output_vector;

	for (int i = 0; i < input_string.size(); i++) {
		output_vector.push_back((uint8_t) input_string[i]);
	}

	return output_vector;
}

std::string StringFromVectorASCII(std::vector<uint8_t> input_vector) {
	std::string out = "";

	for (uint32_t i = 0; i < input_vector.size(); i++) {
		out += (char) input_vector[i];
	}

	return out;
}

std::string StringFromByteVector(std::vector<uint8_t> input_vector, std::string output_type) {
	std::string temp; //needed for upper case hex output
	if (output_type.compare("base64") == 0 || output_type.compare("b64") == 0 || output_type.compare("Base64") == 0 || output_type.compare("B64") == 0) {
		return GetBase64String(input_vector);
	} else if (output_type.compare("hex") == 0 || output_type.compare("h") == 0) { //returns lower case hex output
		return GetHexString(input_vector);
	} else if (output_type.compare("H") == 0 || output_type.compare("Hex") == 0 || output_type.compare("HEX") == 0) { //returns upper case hex output
		temp = GetHexString(input_vector);
		transform(temp.begin(), temp.end(), temp.begin(), ::toupper);
		return temp;
	} else if (output_type.compare("ASCII") == 0 || output_type.compare("ascii") == 0) {
		return StringFromVectorASCII(input_vector);
	} else {
		throw std::invalid_argument("GetStringOutput: unknown output_type.");
	}
}

std::vector<uint8_t> ByteVectorFromString(std::string input_string, std::string input_type) {
	//this function uses an if statement and the hexStringInput function to allow for easy expansion of the MatasanoConverter class
	//e.g. for base64 input
	if (input_type.compare("hex") == 0 || input_type.compare("Hex") == 0 || input_type.compare("h") == 0) {
		return HexStringConvert(input_string);
	} else if (input_type.compare("ASCII") == 0 || input_type.compare("ascii") == 0 || input_type.compare("Ascii")) {
		return ASCIIStringConvert(input_string);
	} else {
		throw std::invalid_argument("inputString: unknown inputType.");
	}
}

<|endoftext|>