<commit_before>/*
 * Copyright (c) 2007 Digital Bazaar, Inc.  All rights reserved.
 */
#include "Thread.h"
#include "System.h"

using namespace std;
using namespace db::rt;

// initialize current thread key parameters
pthread_once_t Thread::CURRENT_THREAD_KEY_INIT = PTHREAD_ONCE_INIT;
pthread_key_t Thread::CURRENT_THREAD_KEY;

// initialize exception key parameters
pthread_once_t Thread::EXCEPTION_KEY_INIT = PTHREAD_ONCE_INIT;
pthread_key_t Thread::EXCEPTION_KEY;

// Note: disabled due to a lack of support in windows
// initialize signal handler parameters
//pthread_once_t Thread::SIGINT_HANDLER_INIT = PTHREAD_ONCE_INIT;

Thread::Thread(Runnable* runnable, std::string name)
{
   // initialize POSIX thread attributes
   pthread_attr_init(&mPThreadAttributes);

   // make thread joinable
   pthread_attr_setdetachstate(&mPThreadAttributes, PTHREAD_CREATE_JOINABLE);

   // make thread cancelable upon joins/waits/etc
   pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
   pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL);

   // thread not waiting to enter a Monitor yet
   mWaitMonitor = NULL;

   // store runnable
   mRunnable = runnable;

   // set name
   mName = name;

   // thread is not alive yet
   mAlive = false;

   // thread not detached yet
   mDetached = false;

   // thread is not interrupted yet
   mInterrupted = false;

   // thread has not joined yet
   mJoined = false;

   // thread is not started yet
   mStarted = false;
}

Thread::~Thread()
{
   // destroy the POSIX thread attributes
   pthread_attr_destroy(&mPThreadAttributes);
}

void Thread::run()
{
   // if a Runnable if available, use it
   if(mRunnable != NULL)
   {
      mRunnable->run();
   }
}

void Thread::createCurrentThreadKey()
{
   // create the thread key for obtaining the current thread
   pthread_key_create(&CURRENT_THREAD_KEY, NULL);
}

void Thread::createExceptionKey()
{
   // create the thread key for obtaining the last thread-local exception
   pthread_key_create(&EXCEPTION_KEY, NULL);
}

// Note: disabled due to a lack of support in windows
//void Thread::installSigIntHandler()
//{
//   // create the SIGINT handler
//   struct sigaction newsa;
//   newsa.sa_handler = handleSigInt;
//   newsa.sa_flags = 0;
//   sigemptyset(&newsa.sa_mask);
//
//   // set the SIGINT handler
//   sigaction(SIGINT, &newsa, NULL);
//}
//
//void Thread::handleSigInt(int signum)
//{
//   // no action is necessary, thread already interrupted
//}

void* Thread::execute(void* thread)
{
   // get the Thread object
   Thread* t = (Thread*)thread;

   // create the current thread key, if not created
   pthread_once(&CURRENT_THREAD_KEY_INIT, Thread::createCurrentThreadKey);

   // create the exception key, if not created
   pthread_once(&EXCEPTION_KEY_INIT, Thread::createExceptionKey);

   // set thread specific data for current thread to the Thread
   pthread_setspecific(CURRENT_THREAD_KEY, t);

   // set thread specific data for exception to NULL (no exception yet)
   pthread_setspecific(EXCEPTION_KEY, NULL);

   // Note: disabled due to a lack of support in windows
   // install signal handler
   //pthread_once(&SIGINT_HANDLER_INIT, Thread::installSigIntHandler);

   // thread is alive
   t->mAlive = true;

   // run the passed thread's run() method
   t->run();

   // thread is no longer alive
   t->mAlive = false;

   // clean up any exception
   setException(NULL);

   // detach thread
   t->detach();

   // exit thread
   pthread_exit(NULL);
   return NULL;
}

bool Thread::start()
{
   bool rval = false;

   if(!hasStarted())
   {
      // create the POSIX thread
      int rc = pthread_create(
         &mPThread, &mPThreadAttributes, execute, (void*)this);

      // if the thread was created successfully, return true
      if(rc == 0)
      {
         // thread has started
         mStarted = true;
         rval = true;
      }
   }

   return rval;
}

// Note: disabled due to lack of support in windows
//void Thread::sendSignal(int signum)
//{
//   lock();
//   {
//      if(hasStarted() && isAlive())
//      {
//         pthread_kill(mPThread, signum);
//      }
//   }
//   unlock();
//}

bool Thread::isAlive()
{
   return mAlive;
}

void Thread::interrupt()
{
   // synchronize
   lock();
   {
      // only interrupt if not already interrupted
      if(!isInterrupted())
      {
         // set interrupted flag
         mInterrupted = true;

         // Note: disabled due to lack of support in windows
         // send SIGINT to thread
         //sendSignal(SIGINT);

         // wake up thread if necessary
         if(mWaitMonitor != NULL)
         {
            mWaitMonitor->signalAll();
         }
      }
   }
   unlock();
}

bool Thread::isInterrupted()
{
   return mInterrupted;
}

bool Thread::hasStarted()
{
   return mStarted;
}

void Thread::join()
{
   bool join = false;

   // synchronize
   lock();
   {
      // check for previous detachments/joins
      if(!mDetached && !mJoined)
      {
         join = true;
         mJoined = true;
      }
   }
   unlock();

   if(join)
   {
      // join thread, wait for it to detach/terminate indefinitely
      int status;
      pthread_join(mPThread, (void **)&status);
   }
}

void Thread::detach()
{
   bool detach = false;

   // synchronize
   lock();
   {
      // check for previous detachments/joins
      if(!mDetached && !mJoined)
      {
         detach = true;
         mDetached = true;
      }
   }
   unlock();

   if(detach)
   {
      // detach thread
      pthread_detach(mPThread);
   }
}

void Thread::setName(string name)
{
   mName = name;
}

const string& Thread::getName()
{
   return mName;
}

Thread* Thread::currentThread()
{
   // get a pointer to the current thread
   return (Thread*)pthread_getspecific(CURRENT_THREAD_KEY);
}

bool Thread::interrupted(bool clear)
{
   bool rval = false;

   // get the current thread's interrupted status
   Thread* t = Thread::currentThread();
   if(t != NULL)
   {
      // synchronize
      t->lock();
      {
         if(t->isInterrupted())
         {
            rval = true;

            if(clear)
            {
               // clear interrupted flag
               t->mInterrupted = false;
            }
         }
      }
      t->unlock();
   }

   return rval;
}

InterruptedException* Thread::sleep(unsigned long time)
{
   InterruptedException* rval = NULL;

   // create a lock object
   Object lock;

   lock.lock();
   {
      // wait on the lock object for the specified time
      rval = lock.wait(time);
   }
   lock.unlock();

   return rval;
}

void Thread::yield()
{
   sched_yield();
}

int Thread::select(
   int nfds, fd_set* readfds, fd_set* writefds, fd_set* exceptfds,
   long long timeout, const sigset_t* sigmask)
{
   int rval = 0;

// Note: disabled due to a lack of support in windows
//   // create timeout
//   struct timeval* tv = NULL;
//   struct timeval to;
//   if(timeout > 0)
//   {
//      // set timeout (1 millisecond is 1000 microseconds)
//      to.tv_sec = timeout / 1000LL;
//      to.tv_usec = (timeout % 1000LL) * 1000LL;
//      tv = &to;
//   }
//
//   // FIXME: signals supposedly don't make select() return in windows
//   // this needs to be tested and potentially remedied somehow
//
//   // FIXME: furthermore, even if we block SIGINT (interruption signal) up
//   // until we reach the select call -- and then unblock right before it
//   // the signal could still sneak in right before select() is called and
//   // control is transferred to the kernel, and therefore we'd handle the
//   // SIGINT before the select() call and select() wouldn't get interrupted
//   // (there is pselect() for doing that unblocking atomically, but
//   // it's UNIX only) -- this may be solved by writing to another file
//   // descriptor when we receive SIGINTs and checking that file descriptor
//   // as well as the one we are waiting on -- but this might not be a
//   // viable solution for windows
//
//   // block SIGINTs
//   blockSignal(SIGINT);
//
//   Thread* t = Thread::currentThread();
//   if(!t->isInterrupted())
//   {
//      // FIXME: pselect() required here to do atomic unblocking & selecting
//
//      // wait for file descriptors to be updated
//      unblockSignal(SIGINT);
//      rval = ::select(nfds, readfds, writefds, exceptfds, timeout);
//      if(rval < 0)
//      {
//         if(errno == EINTR)
//         {
//            // interrupt thread
//            t->interrupt();
//         }
//      }
//   }
//   else
//   {
//      rval = -1;
//      errno = EINTR;
//   }

   // clone file descriptor sets
   fd_set readfds2;
   fd_set writefds2;
   fd_set exceptfds2;

   if(readfds != NULL)
   {
      readfds2 = *readfds;
   }

   if(writefds != NULL)
   {
      writefds2 = *writefds;
   }

   if(exceptfds != NULL)
   {
      exceptfds2 = *exceptfds;
   }

   // keep selecting (polling) until timeout is reached
   long long remaining = (timeout <= 0) ? 1 : timeout;

   struct timeval to;
   if(timeout < 0)
   {
      // create instant timeout (polling)
      to.tv_sec = 0;
      to.tv_usec = 0;
   }
   else
   {
      // create 1 millisecond timeout (1 millisecond is 1000 microseconds)
      to.tv_sec = 0;
      to.tv_usec = 1000LL;
   }

   unsigned long long start = System::getCurrentMilliseconds();
   unsigned long long end;

   Thread* t = Thread::currentThread();
   while(!t->isInterrupted() && remaining > 0 && rval == 0)
   {
      // wait for file descriptors to be updated
      rval = ::select(nfds, readfds, writefds, exceptfds, &to);
      if(rval < 0)
      {
         if(errno == EINTR)
         {
            // interrupt thread
            t->interrupt();
         }
         else if(errno == 0)
         {
            // no error, just timed out
            rval = 0;
         }
      }

      // select() implementation may alter sets or timeout, so reset them
      // if calling select() again
      if(remaining > 0 && rval == 0 && timeout >= 0)
      {
         // reset file descriptor sets
         if(readfds != NULL)
         {
            *readfds = readfds2;
         }

         if(writefds != NULL)
         {
            *writefds = writefds2;
         }

         if(exceptfds != NULL)
         {
            *exceptfds = exceptfds2;
         }

         // reset timeout
         to.tv_sec = 0;
         to.tv_usec = 1000LL;
      }

      if(timeout != 0)
      {
         // decrement remaining time
         end = System::getCurrentMilliseconds();
         remaining -= (end - start);
         start = end;
      }
   }

   if(t->isInterrupted())
   {
      rval = -1;
      errno = EINTR;

      // set interrupted exception
      setException(new InterruptedException(
         "Thread '" + t->getName() + "' interrupted"));
   }

   return rval;
}

void Thread::setException(Exception* e)
{
   if(currentThread() != NULL)
   {
      // get the existing exception for the current thread, if any
      Exception* existing = getException();
      if(existing != e)
      {
         // replace the existing exception
         pthread_setspecific(EXCEPTION_KEY, e);

         if(existing != NULL)
         {
            // delete the old exception
            delete existing;
         }
      }
   }
   else if(e != NULL)
   {
      // delete passed exception, since we are not on a thread
      delete e;
   }
}

Exception* Thread::getException()
{
   Exception* rval = NULL;

   if(currentThread() != NULL)
   {
      // get the exception for the current thread, if any
      rval = (Exception*)pthread_getspecific(EXCEPTION_KEY);
   }

   return rval;
}

bool Thread::hasException()
{
   return getException() != NULL;
}

void Thread::clearException()
{
   setException(NULL);
}

InterruptedException* Thread::waitToEnter(Monitor* m, unsigned long timeout)
{
   InterruptedException* rval = NULL;

   Thread* t = currentThread();
   if(t != NULL)
   {
      // set the current thread's wait monitor
      t->mWaitMonitor = m;

      // get the current time and determine if wait should be indefinite
      unsigned long long past = System::getCurrentMilliseconds();
      unsigned long long present;
      unsigned long long change;
      bool indefinite = (timeout == 0);

      // wait while not interrupted, must wait, and timeout not exhausted
      while(!t->isInterrupted() && m->mustWait() && (indefinite || timeout > 0))
      {
         m->wait(timeout);
         present = System::getCurrentMilliseconds();
         change = present - past;
         past = present;
         timeout -= (change > timeout) ? timeout : change;
      }

      // clear the current thread's wait monitor
      t->mWaitMonitor = NULL;

      // create interrupted exception if interrupted
      if(t->isInterrupted())
      {
         rval = new InterruptedException(
            "Thread '" + t->getName() + "' interrupted");
      }
   }

   // set exception
   setException(rval);

   return rval;
}

// Note: disabled due to a lack of support in windows
//void Thread::setSignalMask(const sigset_t* newmask, sigset_t* oldmask)
//{
//   // set signal mask for this thread
//   pthread_sigmask(SIG_SETMASK, newmask, oldmask);
//}
//
//void Thread::blockSignal(int signum)
//{
//   // unblock signal on this thread
//   sigset_t newset;
//   sigemptyset(&newset);
//   sigaddset(&newset, signum);
//   pthread_sigmask(SIG_BLOCK, &newset, NULL);
//}
//
//void Thread::unblockSignal(int signum)
//{
//   // unblock signal on this thread
//   sigset_t newset;
//   sigemptyset(&newset);
//   sigaddset(&newset, signum);
//   pthread_sigmask(SIG_UNBLOCK, &newset, NULL);
//}
//
//void Thread::setSignalHandler(
//   int signum, const struct sigaction* newaction, struct sigaction* oldaction)
//{
//   // set signal handler
//   sigaction(signum, newaction, oldaction);
//}
<commit_msg>Fixed bug where current thread keys were not initialized for the main thread.<commit_after>/*
 * Copyright (c) 2007 Digital Bazaar, Inc.  All rights reserved.
 */
#include "Thread.h"
#include "System.h"

using namespace std;
using namespace db::rt;

// initialize current thread key parameters
pthread_once_t Thread::CURRENT_THREAD_KEY_INIT = PTHREAD_ONCE_INIT;
pthread_key_t Thread::CURRENT_THREAD_KEY;

// initialize exception key parameters
pthread_once_t Thread::EXCEPTION_KEY_INIT = PTHREAD_ONCE_INIT;
pthread_key_t Thread::EXCEPTION_KEY;

// Note: disabled due to a lack of support in windows
// initialize signal handler parameters
//pthread_once_t Thread::SIGINT_HANDLER_INIT = PTHREAD_ONCE_INIT;

Thread::Thread(Runnable* runnable, std::string name)
{
   // initialize POSIX thread attributes
   pthread_attr_init(&mPThreadAttributes);

   // make thread joinable
   pthread_attr_setdetachstate(&mPThreadAttributes, PTHREAD_CREATE_JOINABLE);

   // make thread cancelable upon joins/waits/etc
   pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
   pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL);

   // thread not waiting to enter a Monitor yet
   mWaitMonitor = NULL;

   // store runnable
   mRunnable = runnable;

   // set name
   mName = name;

   // thread is not alive yet
   mAlive = false;

   // thread not detached yet
   mDetached = false;

   // thread is not interrupted yet
   mInterrupted = false;

   // thread has not joined yet
   mJoined = false;

   // thread is not started yet
   mStarted = false;
}

Thread::~Thread()
{
   // destroy the POSIX thread attributes
   pthread_attr_destroy(&mPThreadAttributes);
}

void Thread::run()
{
   // if a Runnable if available, use it
   if(mRunnable != NULL)
   {
      mRunnable->run();
   }
}

void Thread::createCurrentThreadKey()
{
   // create the thread key for obtaining the current thread
   pthread_key_create(&CURRENT_THREAD_KEY, NULL);
   pthread_setspecific(CURRENT_THREAD_KEY, NULL);
}

void Thread::createExceptionKey()
{
   // create the thread key for obtaining the last thread-local exception
   pthread_key_create(&EXCEPTION_KEY, NULL);
   pthread_setspecific(EXCEPTION_KEY, NULL);
}

// Note: disabled due to a lack of support in windows
//void Thread::installSigIntHandler()
//{
//   // create the SIGINT handler
//   struct sigaction newsa;
//   newsa.sa_handler = handleSigInt;
//   newsa.sa_flags = 0;
//   sigemptyset(&newsa.sa_mask);
//
//   // set the SIGINT handler
//   sigaction(SIGINT, &newsa, NULL);
//}
//
//void Thread::handleSigInt(int signum)
//{
//   // no action is necessary, thread already interrupted
//}

void* Thread::execute(void* thread)
{
   // get the Thread object
   Thread* t = (Thread*)thread;

   // create the current thread key, if not created
   pthread_once(&CURRENT_THREAD_KEY_INIT, Thread::createCurrentThreadKey);

   // create the exception key, if not created
   pthread_once(&EXCEPTION_KEY_INIT, Thread::createExceptionKey);

   // set thread specific data for current thread to the Thread
   pthread_setspecific(CURRENT_THREAD_KEY, t);

   // set thread specific data for exception to NULL (no exception yet)
   pthread_setspecific(EXCEPTION_KEY, NULL);

   // Note: disabled due to a lack of support in windows
   // install signal handler
   //pthread_once(&SIGINT_HANDLER_INIT, Thread::installSigIntHandler);

   // thread is alive
   t->mAlive = true;

   // run the passed thread's run() method
   t->run();

   // thread is no longer alive
   t->mAlive = false;

   // clean up any exception
   setException(NULL);

   // detach thread
   t->detach();

   // exit thread
   pthread_exit(NULL);
   return NULL;
}

bool Thread::start()
{
   bool rval = false;

   if(!hasStarted())
   {
      // create the POSIX thread
      int rc = pthread_create(
         &mPThread, &mPThreadAttributes, execute, (void*)this);

      // if the thread was created successfully, return true
      if(rc == 0)
      {
         // thread has started
         mStarted = true;
         rval = true;
      }
   }

   return rval;
}

// Note: disabled due to lack of support in windows
//void Thread::sendSignal(int signum)
//{
//   lock();
//   {
//      if(hasStarted() && isAlive())
//      {
//         pthread_kill(mPThread, signum);
//      }
//   }
//   unlock();
//}

bool Thread::isAlive()
{
   return mAlive;
}

void Thread::interrupt()
{
   // synchronize
   lock();
   {
      // only interrupt if not already interrupted
      if(!isInterrupted())
      {
         // set interrupted flag
         mInterrupted = true;

         // Note: disabled due to lack of support in windows
         // send SIGINT to thread
         //sendSignal(SIGINT);

         // wake up thread if necessary
         if(mWaitMonitor != NULL)
         {
            mWaitMonitor->signalAll();
         }
      }
   }
   unlock();
}

bool Thread::isInterrupted()
{
   return mInterrupted;
}

bool Thread::hasStarted()
{
   return mStarted;
}

void Thread::join()
{
   bool join = false;

   // synchronize
   lock();
   {
      // check for previous detachments/joins
      if(!mDetached && !mJoined)
      {
         join = true;
         mJoined = true;
      }
   }
   unlock();

   if(join)
   {
      // join thread, wait for it to detach/terminate indefinitely
      int status;
      pthread_join(mPThread, (void **)&status);
   }
}

void Thread::detach()
{
   bool detach = false;

   // synchronize
   lock();
   {
      // check for previous detachments/joins
      if(!mDetached && !mJoined)
      {
         detach = true;
         mDetached = true;
      }
   }
   unlock();

   if(detach)
   {
      // detach thread
      pthread_detach(mPThread);
   }
}

void Thread::setName(string name)
{
   mName = name;
}

const string& Thread::getName()
{
   return mName;
}

Thread* Thread::currentThread()
{
   // create the current thread key, if not created
   pthread_once(&CURRENT_THREAD_KEY_INIT, Thread::createCurrentThreadKey);

   // get a pointer to the current thread
   return (Thread*)pthread_getspecific(CURRENT_THREAD_KEY);
}

bool Thread::interrupted(bool clear)
{
   bool rval = false;

   // get the current thread's interrupted status
   Thread* t = Thread::currentThread();
   if(t != NULL)
   {
      // synchronize
      t->lock();
      {
         if(t->isInterrupted())
         {
            rval = true;

            if(clear)
            {
               // clear interrupted flag
               t->mInterrupted = false;
            }
         }
      }
      t->unlock();
   }

   return rval;
}

InterruptedException* Thread::sleep(unsigned long time)
{
   InterruptedException* rval = NULL;

   // create a lock object
   Object lock;

   lock.lock();
   {
      // wait on the lock object for the specified time
      rval = lock.wait(time);
   }
   lock.unlock();

   return rval;
}

void Thread::yield()
{
   sched_yield();
}

int Thread::select(
   int nfds, fd_set* readfds, fd_set* writefds, fd_set* exceptfds,
   long long timeout, const sigset_t* sigmask)
{
   int rval = 0;

// Note: disabled due to a lack of support in windows
//   // create timeout
//   struct timeval* tv = NULL;
//   struct timeval to;
//   if(timeout > 0)
//   {
//      // set timeout (1 millisecond is 1000 microseconds)
//      to.tv_sec = timeout / 1000LL;
//      to.tv_usec = (timeout % 1000LL) * 1000LL;
//      tv = &to;
//   }
//
//   // FIXME: signals supposedly don't make select() return in windows
//   // this needs to be tested and potentially remedied somehow
//
//   // FIXME: furthermore, even if we block SIGINT (interruption signal) up
//   // until we reach the select call -- and then unblock right before it
//   // the signal could still sneak in right before select() is called and
//   // control is transferred to the kernel, and therefore we'd handle the
//   // SIGINT before the select() call and select() wouldn't get interrupted
//   // (there is pselect() for doing that unblocking atomically, but
//   // it's UNIX only) -- this may be solved by writing to another file
//   // descriptor when we receive SIGINTs and checking that file descriptor
//   // as well as the one we are waiting on -- but this might not be a
//   // viable solution for windows
//
//   // block SIGINTs
//   blockSignal(SIGINT);
//
//   Thread* t = Thread::currentThread();
//   if(!t->isInterrupted())
//   {
//      // FIXME: pselect() required here to do atomic unblocking & selecting
//
//      // wait for file descriptors to be updated
//      unblockSignal(SIGINT);
//      rval = ::select(nfds, readfds, writefds, exceptfds, timeout);
//      if(rval < 0)
//      {
//         if(errno == EINTR)
//         {
//            // interrupt thread
//            t->interrupt();
//         }
//      }
//   }
//   else
//   {
//      rval = -1;
//      errno = EINTR;
//   }

   // clone file descriptor sets
   fd_set readfds2;
   fd_set writefds2;
   fd_set exceptfds2;

   if(readfds != NULL)
   {
      readfds2 = *readfds;
   }

   if(writefds != NULL)
   {
      writefds2 = *writefds;
   }

   if(exceptfds != NULL)
   {
      exceptfds2 = *exceptfds;
   }

   // keep selecting (polling) until timeout is reached
   long long remaining = (timeout <= 0) ? 1 : timeout;

   struct timeval to;
   if(timeout < 0)
   {
      // create instant timeout (polling)
      to.tv_sec = 0;
      to.tv_usec = 0;
   }
   else
   {
      // create 1 millisecond timeout (1 millisecond is 1000 microseconds)
      to.tv_sec = 0;
      to.tv_usec = 1000LL;
   }

   unsigned long long start = System::getCurrentMilliseconds();
   unsigned long long end;

   Thread* t = Thread::currentThread();
   while(!t->isInterrupted() && remaining > 0 && rval == 0)
   {
      // wait for file descriptors to be updated
      rval = ::select(nfds, readfds, writefds, exceptfds, &to);
      if(rval < 0)
      {
         if(errno == EINTR)
         {
            // interrupt thread
            t->interrupt();
         }
         else if(errno == 0)
         {
            // no error, just timed out
            rval = 0;
         }
      }

      // select() implementation may alter sets or timeout, so reset them
      // if calling select() again
      if(remaining > 0 && rval == 0 && timeout >= 0)
      {
         // reset file descriptor sets
         if(readfds != NULL)
         {
            *readfds = readfds2;
         }

         if(writefds != NULL)
         {
            *writefds = writefds2;
         }

         if(exceptfds != NULL)
         {
            *exceptfds = exceptfds2;
         }

         // reset timeout
         to.tv_sec = 0;
         to.tv_usec = 1000LL;
      }

      if(timeout != 0)
      {
         // decrement remaining time
         end = System::getCurrentMilliseconds();
         remaining -= (end - start);
         start = end;
      }
   }

   if(t->isInterrupted())
   {
      rval = -1;
      errno = EINTR;

      // set interrupted exception
      setException(new InterruptedException(
         "Thread '" + t->getName() + "' interrupted"));
   }

   return rval;
}

void Thread::setException(Exception* e)
{
   if(currentThread() != NULL)
   {
      // get the existing exception for the current thread, if any
      Exception* existing = getException();
      if(existing != e)
      {
         // replace the existing exception
         pthread_setspecific(EXCEPTION_KEY, e);

         if(existing != NULL)
         {
            // delete the old exception
            delete existing;
         }
      }
   }
   else if(e != NULL)
   {
      // delete passed exception, since we are not on a thread
      delete e;
   }
}

Exception* Thread::getException()
{
   Exception* rval = NULL;

   // create the exception key, if not created
   pthread_once(&EXCEPTION_KEY_INIT, Thread::createExceptionKey);

   if(currentThread() != NULL)
   {
      // get the exception for the current thread, if any
      rval = (Exception*)pthread_getspecific(EXCEPTION_KEY);
   }

   return rval;
}

bool Thread::hasException()
{
   return getException() != NULL;
}

void Thread::clearException()
{
   setException(NULL);
}

InterruptedException* Thread::waitToEnter(Monitor* m, unsigned long timeout)
{
   InterruptedException* rval = NULL;

   Thread* t = currentThread();
   if(t != NULL)
   {
      // set the current thread's wait monitor
      t->mWaitMonitor = m;

      // get the current time and determine if wait should be indefinite
      unsigned long long past = System::getCurrentMilliseconds();
      unsigned long long present;
      unsigned long long change;
      bool indefinite = (timeout == 0);

      // wait while not interrupted, must wait, and timeout not exhausted
      while(!t->isInterrupted() && m->mustWait() && (indefinite || timeout > 0))
      {
         m->wait(timeout);
         present = System::getCurrentMilliseconds();
         change = present - past;
         past = present;
         timeout -= (change > timeout) ? timeout : change;
      }

      // clear the current thread's wait monitor
      t->mWaitMonitor = NULL;

      // create interrupted exception if interrupted
      if(t->isInterrupted())
      {
         rval = new InterruptedException(
            "Thread '" + t->getName() + "' interrupted");
      }
   }

   // set exception
   setException(rval);

   return rval;
}

// Note: disabled due to a lack of support in windows
//void Thread::setSignalMask(const sigset_t* newmask, sigset_t* oldmask)
//{
//   // set signal mask for this thread
//   pthread_sigmask(SIG_SETMASK, newmask, oldmask);
//}
//
//void Thread::blockSignal(int signum)
//{
//   // unblock signal on this thread
//   sigset_t newset;
//   sigemptyset(&newset);
//   sigaddset(&newset, signum);
//   pthread_sigmask(SIG_BLOCK, &newset, NULL);
//}
//
//void Thread::unblockSignal(int signum)
//{
//   // unblock signal on this thread
//   sigset_t newset;
//   sigemptyset(&newset);
//   sigaddset(&newset, signum);
//   pthread_sigmask(SIG_UNBLOCK, &newset, NULL);
//}
//
//void Thread::setSignalHandler(
//   int signum, const struct sigaction* newaction, struct sigaction* oldaction)
//{
//   // set signal handler
//   sigaction(signum, newaction, oldaction);
//}
<|endoftext|>


<commit_before>#include "Basis.h"

Basis::Basis(void)
{
	_error = true;
	_warning = true;
	_info = false;
	_debug = false;
	_debugReport = false;
	_bugReportFileName = "BugReport";
}

Basis::~Basis(void)
{
	;
}

void Basis::setSourceFileName(std::string pSourceFileName){
	//pSourceFileName.replace(0,2,"");	//get rid of the .\ in the file name
	pSourceFileName = pSourceFileName.substr(0,pSourceFileName.find_last_of("."));		//get rid of the .cxx in the file name
	_sourceFileName = pSourceFileName;
}

void Basis::setErrorOutput(bool pToggle)
{
	_error = pToggle;
}
void Basis::setWarningOutput(bool pToggle)
{
	_warning = pToggle;
}
void Basis::setInfoOutput(bool pToggle)
{
	_info = pToggle;
}
void Basis::setDebugOutput(bool pToggle)
{
	_debug = pToggle;
}

void Basis::debug(std::string& pText, int pLine)
{
  if(_debug){
	  std::stringstream tOutString;
	  if (pLine == -1)
		  tOutString<<"DEBUG "<<_sourceFileName<<"::"<<pText;
	  else
		  tOutString<<"DEBUG "<<_sourceFileName<<"("<<pLine<<")::";
	  std::cout<<tOutString.str()<<"\n";
	  if (_debugReport){
		  std::ofstream tBugReport;
		  tBugReport.open(_bugReportFileName.c_str(), std::ios::out | std::ios::app);
		  tBugReport<<tOutString.str()<<std::endl;
		  tBugReport.close();
	  }
  }
}

void Basis::info(std::string& pText, int pLine)
{
  if(_info){
	  std::stringstream tOutString;
	  if (pLine == -1)
		  tOutString<<"INFO "<<_sourceFileName<<"::"<<pText;
	  else
		  tOutString<<"INFO "<<_sourceFileName<<"("<<pLine<<")::"<<pText;
	  std::cout<<tOutString.str()<<"\n";
	  if (_debugReport){
		  std::ofstream tBugReport;
		  tBugReport.open(_bugReportFileName.c_str(), std::ios::out | std::ios::app);
		  tBugReport<<tOutString.str()<<std::endl;
		  tBugReport.close();
	  }
  }
}

void Basis::warning(std::string& pText, int pLine)
{
		std::stringstream tOutString;
		if (pLine == -1)
			tOutString<<"WARNING "<<_sourceFileName<<"::"<<pText;
		else
			tOutString<<"WARNING "<<_sourceFileName<<"("<<pLine<<")::"<<pText;
		std::cout<<tOutString.str()<<"\n";
		if (_debugReport){
			std::ofstream tBugReport;
			tBugReport.open(_bugReportFileName.c_str(), std::ios::out | std::ios::app);
			tBugReport<<tOutString.str()<<std::endl;
			tBugReport.close();
		}
}

void Basis::error(std::string& pText, int pLine)
{
	std::stringstream tOutString;
	if (pLine == -1)
		tOutString<<"ERROR "<<_sourceFileName<<"::"<<pText;
	else
		tOutString<<"ERROR "<<_sourceFileName<<"("<<pLine<<")::"<<pText;
	std::cout<<tOutString.str()<<"\n";
	if (_debugReport){
		std::ofstream tBugReport;
		tBugReport.open(_bugReportFileName.c_str(), std::ios::out | std::ios::app);
		tBugReport<<tOutString.str()<<std::endl;
		tBugReport.close();
	}
}
bool Basis::getStringSeparated(std::string pLine, std::string pSeparator, std::string& pLeft, std::string& pRight)
{
	int tFound = 0;
	tFound = pLine.find_first_of(pSeparator);
	if(tFound != pLine.npos){ //abort if no seperator found
		pLeft = pLine.substr(0, tFound);
		pRight = pLine.substr(tFound+pSeparator.size(), pLine.npos);
		return true;
	}
	return false;
}

bool Basis::isInf(double pValue)
{
	return std::numeric_limits<double>::has_infinity && pValue == std::numeric_limits<double>::infinity();
}

bool Basis::isNan(double pValue)
{
	return pValue != pValue;
}
bool Basis::isFinite(double pValue)
{
	return !isInf(pValue) &&  !isNan(pValue);
}

bool Basis::fileExists(const std::string& pFileName)
{
	std::ifstream tFile(pFileName.c_str());
	return (tFile != 0);
}

double Basis::StrToDouble(std::string const& pValue)
{
	std::istringstream tValue(pValue);
	double tDoubleValue;
	if (!(tValue>>tDoubleValue)){
		error(std::string("StrToDouble(std::string const& pValue): Not a valid double value set: ").append(pValue));
		return -1;
	}
	return tDoubleValue;
}

int Basis::StrToInt(std::string const& pValue)
{
	std::istringstream tValue(pValue);
	int tIntValue;
	if (!(tValue>>tIntValue)){
		error(std::string("StrToInt(std::string const& pValue): Not a valid integer value set: ").append(pValue));
		return 0;
	}
	return tIntValue;
}

std::string Basis::IntToStr(unsigned int const& pValue)
{
	std::stringstream tStream;
	tStream << pValue;
	return tStream.str();
}

std::string Basis::DoubleToStr(double const& pValue)
{
	std::stringstream tValue;
	tValue << pValue;
	return tValue.str();
}

std::string Basis::IntToBin(unsigned int pValue)
{
	std::string tResult = "";
	do
	{
		if ( (pValue & 1) == 0 )
			tResult += "0";
		else
			tResult += "1";
		pValue >>= 1;
	} while (pValue);

	std::reverse(tResult.begin(), tResult.end());
	return tResult;
}

void Basis::setBugReport(bool pCreateReport)
{
	_debugReport = pCreateReport;
}
<commit_msg>- warnings can be turned off now (bug fix)<commit_after>#include "Basis.h"

Basis::Basis(void)
{
	_error = true;
	_warning = true;
	_info = false;
	_debug = false;
	_debugReport = false;
	_bugReportFileName = "BugReport";
}

Basis::~Basis(void)
{
	;
}

void Basis::setSourceFileName(std::string pSourceFileName){
	//pSourceFileName.replace(0,2,"");	//get rid of the .\ in the file name
	pSourceFileName = pSourceFileName.substr(0,pSourceFileName.find_last_of("."));		//get rid of the .cxx in the file name
	_sourceFileName = pSourceFileName;
}

void Basis::setErrorOutput(bool pToggle)
{
	_error = pToggle;
}
void Basis::setWarningOutput(bool pToggle)
{
	_warning = pToggle;
}
void Basis::setInfoOutput(bool pToggle)
{
	_info = pToggle;
}
void Basis::setDebugOutput(bool pToggle)
{
	_debug = pToggle;
}

void Basis::debug(std::string& pText, int pLine)
{
  if(_debug){
	  std::stringstream tOutString;
	  if (pLine == -1)
		  tOutString<<"DEBUG "<<_sourceFileName<<"::"<<pText;
	  else
		  tOutString<<"DEBUG "<<_sourceFileName<<"("<<pLine<<")::";
	  std::cout<<tOutString.str()<<"\n";
	  if (_debugReport){
		  std::ofstream tBugReport;
		  tBugReport.open(_bugReportFileName.c_str(), std::ios::out | std::ios::app);
		  tBugReport<<tOutString.str()<<std::endl;
		  tBugReport.close();
	  }
  }
}

void Basis::info(std::string& pText, int pLine)
{
  if(_info){
	  std::stringstream tOutString;
	  if (pLine == -1)
		  tOutString<<"INFO "<<_sourceFileName<<"::"<<pText;
	  else
		  tOutString<<"INFO "<<_sourceFileName<<"("<<pLine<<")::"<<pText;
	  std::cout<<tOutString.str()<<"\n";
	  if (_debugReport){
		  std::ofstream tBugReport;
		  tBugReport.open(_bugReportFileName.c_str(), std::ios::out | std::ios::app);
		  tBugReport<<tOutString.str()<<std::endl;
		  tBugReport.close();
	  }
  }
}

void Basis::warning(std::string& pText, int pLine)
{
  if(_warning){
		std::stringstream tOutString;
		if (pLine == -1)
			tOutString<<"WARNING "<<_sourceFileName<<"::"<<pText;
		else
			tOutString<<"WARNING "<<_sourceFileName<<"("<<pLine<<")::"<<pText;
		std::cout<<tOutString.str()<<"\n";
		if (_debugReport){
			std::ofstream tBugReport;
			tBugReport.open(_bugReportFileName.c_str(), std::ios::out | std::ios::app);
			tBugReport<<tOutString.str()<<std::endl;
			tBugReport.close();
		}
  }
}

void Basis::error(std::string& pText, int pLine)
{
	std::stringstream tOutString;
	if (pLine == -1)
		tOutString<<"ERROR "<<_sourceFileName<<"::"<<pText;
	else
		tOutString<<"ERROR "<<_sourceFileName<<"("<<pLine<<")::"<<pText;
	std::cout<<tOutString.str()<<"\n";
	if (_debugReport){
		std::ofstream tBugReport;
		tBugReport.open(_bugReportFileName.c_str(), std::ios::out | std::ios::app);
		tBugReport<<tOutString.str()<<std::endl;
		tBugReport.close();
	}
}
bool Basis::getStringSeparated(std::string pLine, std::string pSeparator, std::string& pLeft, std::string& pRight)
{
	int tFound = 0;
	tFound = pLine.find_first_of(pSeparator);
	if(tFound != pLine.npos){ //abort if no seperator found
		pLeft = pLine.substr(0, tFound);
		pRight = pLine.substr(tFound+pSeparator.size(), pLine.npos);
		return true;
	}
	return false;
}

bool Basis::isInf(double pValue)
{
	return std::numeric_limits<double>::has_infinity && pValue == std::numeric_limits<double>::infinity();
}

bool Basis::isNan(double pValue)
{
	return pValue != pValue;
}
bool Basis::isFinite(double pValue)
{
	return !isInf(pValue) &&  !isNan(pValue);
}

bool Basis::fileExists(const std::string& pFileName)
{
	std::ifstream tFile(pFileName.c_str());
	return (tFile != 0);
}

double Basis::StrToDouble(std::string const& pValue)
{
	std::istringstream tValue(pValue);
	double tDoubleValue;
	if (!(tValue>>tDoubleValue)){
		error(std::string("StrToDouble(std::string const& pValue): Not a valid double value set: ").append(pValue));
		return -1;
	}
	return tDoubleValue;
}

int Basis::StrToInt(std::string const& pValue)
{
	std::istringstream tValue(pValue);
	int tIntValue;
	if (!(tValue>>tIntValue)){
		error(std::string("StrToInt(std::string const& pValue): Not a valid integer value set: ").append(pValue));
		return 0;
	}
	return tIntValue;
}

std::string Basis::IntToStr(unsigned int const& pValue)
{
	std::stringstream tStream;
	tStream << pValue;
	return tStream.str();
}

std::string Basis::DoubleToStr(double const& pValue)
{
	std::stringstream tValue;
	tValue << pValue;
	return tValue.str();
}

std::string Basis::IntToBin(unsigned int pValue)
{
	std::string tResult = "";
	do
	{
		if ( (pValue & 1) == 0 )
			tResult += "0";
		else
			tResult += "1";
		pValue >>= 1;
	} while (pValue);

	std::reverse(tResult.begin(), tResult.end());
	return tResult;
}

void Basis::setBugReport(bool pCreateReport)
{
	_debugReport = pCreateReport;
}
<|endoftext|>


<commit_before>#include <sstream>
#include <iostream>
#include <vector>
#include <cassert>
#include <cmath>
#include <tr1/memory>

#include <boost/program_options.hpp>
#include <boost/program_options/variables_map.hpp>

#include "stringlib.h"
#include "hg_sampler.h"
#include "sentence_metadata.h"
#include "ns.h"
#include "ns_docscorer.h"
#include "verbose.h"
#include "viterbi.h"
#include "hg.h"
#include "prob.h"
#include "kbest.h"
#include "ff_register.h"
#include "decoder.h"
#include "filelib.h"
#include "fdict.h"
#include "weights.h"
#include "sparse_vector.h"
#include "sampler.h"

using namespace std;
namespace po = boost::program_options;

bool invert_score;
std::tr1::shared_ptr<MT19937> rng;

void RandomPermutation(int len, vector<int>* p_ids) {
  vector<int>& ids = *p_ids;
  ids.resize(len);
  for (int i = 0; i < len; ++i) ids[i] = i;
  for (int i = len; i > 0; --i) {
    int j = rng->next() * i;
    if (j == i) i--;
    swap(ids[i-1], ids[j]);
  }
}

bool InitCommandLine(int argc, char** argv, po::variables_map* conf) {
  po::options_description opts("Configuration options");
  opts.add_options()
        ("input_weights,w",po::value<string>(),"Input feature weights file")
        ("source,i",po::value<string>(),"Source file for development set")
        ("passes,p", po::value<int>()->default_value(15), "Number of passes through the training data")
        ("reference,r",po::value<vector<string> >(), "[REQD] Reference translation(s) (tokenized text file)")
        ("mt_metric,m",po::value<string>()->default_value("ibm_bleu"), "Scoring metric (ibm_bleu, nist_bleu, koehn_bleu, ter, combi)")
        ("max_step_size,C", po::value<double>()->default_value(0.01), "regularization strength (C)")
        ("mt_metric_scale,s", po::value<double>()->default_value(1.0), "Amount to scale MT loss function by")
        ("k_best_size,k", po::value<int>()->default_value(250), "Size of hypothesis list to search for oracles")
        ("sample_forest,f", "Instead of a k-best list, sample k hypotheses from the decoder's forest")
        ("sample_forest_unit_weight_vector,x", "Before sampling (must use -f option), rescale the weight vector used so it has unit length; this may improve the quality of the samples")
        ("random_seed,S", po::value<uint32_t>(), "Random seed (if not specified, /dev/random will be used)")
        ("decoder_config,c",po::value<string>(),"Decoder configuration file");
  po::options_description clo("Command line options");
  clo.add_options()
        ("config", po::value<string>(), "Configuration file")
        ("help,h", "Print this help message and exit");
  po::options_description dconfig_options, dcmdline_options;
  dconfig_options.add(opts);
  dcmdline_options.add(opts).add(clo);

  po::store(parse_command_line(argc, argv, dcmdline_options), *conf);
  if (conf->count("config")) {
    ifstream config((*conf)["config"].as<string>().c_str());
    po::store(po::parse_config_file(config, dconfig_options), *conf);
  }
  po::notify(*conf);

  if (conf->count("help") || !conf->count("input_weights") || !conf->count("source") || !conf->count("decoder_config") || !conf->count("reference")) {
    cerr << dcmdline_options << endl;
    return false;
  }
  return true;
}

static const double kMINUS_EPSILON = -1e-6;

struct HypothesisInfo {
  SparseVector<double> features;
  double mt_metric;
};

struct GoodBadOracle {
  std::tr1::shared_ptr<HypothesisInfo> good;
  std::tr1::shared_ptr<HypothesisInfo> bad;
};

struct TrainingObserver : public DecoderObserver {
  TrainingObserver(const int k, const DocumentScorer& d, const EvaluationMetric& m, bool sf, vector<GoodBadOracle>* o) : ds(d), metric(m), oracles(*o), kbest_size(k), sample_forest(sf) {}
  const DocumentScorer& ds;
  const EvaluationMetric& metric;
  vector<GoodBadOracle>& oracles;
  std::tr1::shared_ptr<HypothesisInfo> cur_best;
  const int kbest_size;
  const bool sample_forest;

  const HypothesisInfo& GetCurrentBestHypothesis() const {
    return *cur_best;
  }

  virtual void NotifyTranslationForest(const SentenceMetadata& smeta, Hypergraph* hg) {
    UpdateOracles(smeta.GetSentenceID(), *hg);
  }

  std::tr1::shared_ptr<HypothesisInfo> MakeHypothesisInfo(const SparseVector<double>& feats, const double score) {
    std::tr1::shared_ptr<HypothesisInfo> h(new HypothesisInfo);
    h->features = feats;
    h->mt_metric = score;
    return h;
  }

  void UpdateOracles(int sent_id, const Hypergraph& forest) {
    std::tr1::shared_ptr<HypothesisInfo>& cur_good = oracles[sent_id].good;
    std::tr1::shared_ptr<HypothesisInfo>& cur_bad = oracles[sent_id].bad;
    cur_bad.reset();  // TODO get rid of??

    if (sample_forest) {
      vector<WordID> cur_prediction;
      ViterbiESentence(forest, &cur_prediction);
      SufficientStats sstats;
      ds[sent_id]->Evaluate(cur_prediction, &sstats);
      float sentscore = metric.ComputeScore(sstats);
      cur_best = MakeHypothesisInfo(ViterbiFeatures(forest), sentscore);

      vector<HypergraphSampler::Hypothesis> samples;
      HypergraphSampler::sample_hypotheses(forest, kbest_size, &*rng, &samples);
      for (unsigned i = 0; i < samples.size(); ++i) {
        ds[sent_id]->Evaluate(samples[i].words, &sstats);
        float sentscore = metric.ComputeScore(sstats);
        if (invert_score) sentscore *= -1.0;
        if (!cur_good || sentscore > cur_good->mt_metric)
          cur_good = MakeHypothesisInfo(samples[i].fmap, sentscore);
        if (!cur_bad || sentscore < cur_bad->mt_metric)
          cur_bad = MakeHypothesisInfo(samples[i].fmap, sentscore);
      }
    } else {
      KBest::KBestDerivations<vector<WordID>, ESentenceTraversal> kbest(forest, kbest_size);
      SufficientStats sstats;
      for (int i = 0; i < kbest_size; ++i) {
        const KBest::KBestDerivations<vector<WordID>, ESentenceTraversal>::Derivation* d =
          kbest.LazyKthBest(forest.nodes_.size() - 1, i);
        if (!d) break;
        ds[sent_id]->Evaluate(d->yield, &sstats);
        float sentscore = metric.ComputeScore(sstats);
        if (invert_score) sentscore *= -1.0;
        // cerr << TD::GetString(d->yield) << " ||| " << d->score << " ||| " << sentscore << endl;
        if (i == 0)
          cur_best = MakeHypothesisInfo(d->feature_values, sentscore);
        if (!cur_good || sentscore > cur_good->mt_metric)
          cur_good = MakeHypothesisInfo(d->feature_values, sentscore);
        if (!cur_bad || sentscore < cur_bad->mt_metric)
          cur_bad = MakeHypothesisInfo(d->feature_values, sentscore);
      }
      //cerr << "GOOD: " << cur_good->mt_metric << endl;
      //cerr << " CUR: " << cur_best->mt_metric << endl;
      //cerr << " BAD: " << cur_bad->mt_metric << endl;
    }
  }
};

void ReadTrainingCorpus(const string& fname, vector<string>* c) {
  ReadFile rf(fname);
  istream& in = *rf.stream();
  string line;
  while(in) {
    getline(in, line);
    if (!in) break;
    c->push_back(line);
  }
}

bool ApproxEqual(double a, double b) {
  if (a == b) return true;
  return (fabs(a-b)/fabs(b)) < 0.000001;
}

int main(int argc, char** argv) {
  register_feature_functions();
  SetSilent(true);  // turn off verbose decoder output

  po::variables_map conf;
  if (!InitCommandLine(argc, argv, &conf)) return 1;

  if (conf.count("random_seed"))
    rng.reset(new MT19937(conf["random_seed"].as<uint32_t>()));
  else
    rng.reset(new MT19937);
  const bool sample_forest = conf.count("sample_forest") > 0;
  const bool sample_forest_unit_weight_vector = conf.count("sample_forest_unit_weight_vector") > 0;
  if (sample_forest_unit_weight_vector && !sample_forest) {
    cerr << "Cannot --sample_forest_unit_weight_vector without --sample_forest" << endl;
    return 1;
  }
  vector<string> corpus;
  ReadTrainingCorpus(conf["source"].as<string>(), &corpus);

  string metric_name = UppercaseString(conf["evaluation_metric"].as<string>());
  if (metric_name == "COMBI") {
    cerr << "WARNING: 'combi' metric is no longer supported, switching to 'COMB:TER=-0.5;IBM_BLEU=0.5'\n";
    metric_name = "COMB:TER=-0.5;IBM_BLEU=0.5";
  } else if (metric_name == "BLEU") {
    cerr << "WARNING: 'BLEU' is ambiguous, assuming 'IBM_BLEU'\n";
    metric_name = "IBM_BLEU";
  }
  EvaluationMetric* metric = EvaluationMetric::Instance(metric_name);
  DocumentScorer ds(metric, conf["reference"].as<vector<string> >());
  cerr << "Loaded " << ds.size() << " references for scoring with " << metric_name << endl;
  invert_score = metric->IsErrorMetric();

  if (ds.size() != corpus.size()) {
    cerr << "Mismatched number of references (" << ds.size() << ") and sources (" << corpus.size() << ")\n";
    return 1;
  }

  ReadFile ini_rf(conf["decoder_config"].as<string>());
  Decoder decoder(ini_rf.stream());

  // load initial weights
  vector<weight_t>& dense_weights = decoder.CurrentWeightVector();
  SparseVector<weight_t> lambdas;
  Weights::InitFromFile(conf["input_weights"].as<string>(), &dense_weights);
  Weights::InitSparseVector(dense_weights, &lambdas);

  const double max_step_size = conf["max_step_size"].as<double>();
  const double mt_metric_scale = conf["mt_metric_scale"].as<double>();

  assert(corpus.size() > 0);
  vector<GoodBadOracle> oracles(corpus.size());

  TrainingObserver observer(conf["k_best_size"].as<int>(), ds, *metric, sample_forest, &oracles);
  int cur_sent = 0;
  int lcount = 0;
  int normalizer = 0;
  double tot_loss = 0;
  int dots = 0;
  int cur_pass = 0;
  SparseVector<double> tot;
  tot += lambdas;          // initial weights
  normalizer++;            // count for initial weights
  int max_iteration = conf["passes"].as<int>() * corpus.size();
  string msg = "# MIRA tuned weights";
  string msga = "# MIRA tuned weights AVERAGED";
  vector<int> order;
  RandomPermutation(corpus.size(), &order);
  while (lcount <= max_iteration) {
    lambdas.init_vector(&dense_weights);
    if ((cur_sent * 40 / corpus.size()) > dots) { ++dots; cerr << '.'; }
    if (corpus.size() == cur_sent) {
      cerr << " [AVG METRIC LAST PASS=" << (tot_loss / corpus.size()) << "]\n";
      Weights::ShowLargestFeatures(dense_weights);
      cur_sent = 0;
      tot_loss = 0;
      dots = 0;
      ostringstream os;
      os << "weights.mira-pass" << (cur_pass < 10 ? "0" : "") << cur_pass << ".gz";
      SparseVector<double> x = tot;
      x /= normalizer;
      ostringstream sa;
      sa << "weights.mira-pass" << (cur_pass < 10 ? "0" : "") << cur_pass << "-avg.gz";
      x.init_vector(&dense_weights);
      Weights::WriteToFile(os.str(), dense_weights, true, &msg);
      ++cur_pass;
      RandomPermutation(corpus.size(), &order);
    }
    if (cur_sent == 0) {
      cerr << "PASS " << (lcount / corpus.size() + 1) << endl;
    }
    decoder.SetId(order[cur_sent]);
    double sc = 1.0;
    if (sample_forest_unit_weight_vector) {
      sc = lambdas.l2norm();
      if (sc > 0) {
        for (unsigned i = 0; i < dense_weights.size(); ++i)
          dense_weights[i] /= sc;
      }
    }
    decoder.Decode(corpus[order[cur_sent]], &observer);  // update oracles
    if (sc && sc != 1.0) {
      for (unsigned i = 0; i < dense_weights.size(); ++i)
        dense_weights[i] *= sc;
    }
    const HypothesisInfo& cur_hyp = observer.GetCurrentBestHypothesis();
    const HypothesisInfo& cur_good = *oracles[order[cur_sent]].good;
    const HypothesisInfo& cur_bad = *oracles[order[cur_sent]].bad;
    tot_loss += cur_hyp.mt_metric;
    if (!ApproxEqual(cur_hyp.mt_metric, cur_good.mt_metric)) {
      const double loss = cur_bad.features.dot(dense_weights) - cur_good.features.dot(dense_weights) +
          mt_metric_scale * (cur_good.mt_metric - cur_bad.mt_metric);
      //cerr << "LOSS: " << loss << endl;
      if (loss > 0.0) {
        SparseVector<double> diff = cur_good.features;
        diff -= cur_bad.features;
        double step_size = loss / diff.l2norm_sq();
        //cerr << loss << " " << step_size << " " << diff << endl;
        if (step_size > max_step_size) step_size = max_step_size;
        lambdas += (cur_good.features * step_size);
        lambdas -= (cur_bad.features * step_size);
        //cerr << "L: " << lambdas << endl;
      }
    }
    tot += lambdas;
    ++normalizer;
    ++lcount;
    ++cur_sent;
  }
  cerr << endl;
  Weights::WriteToFile("weights.mira-final.gz", dense_weights, true, &msg);
  tot /= normalizer;
  tot.init_vector(dense_weights);
  msg = "# MIRA tuned weights (averaged vector)";
  Weights::WriteToFile("weights.mira-final-avg.gz", dense_weights, true, &msg);
  cerr << "Optimization complete.\nAVERAGED WEIGHTS: weights.mira-final-avg.gz\n";
  return 0;
}

<commit_msg>bug fix<commit_after>#include <sstream>
#include <iostream>
#include <vector>
#include <cassert>
#include <cmath>
#include <tr1/memory>

#include <boost/program_options.hpp>
#include <boost/program_options/variables_map.hpp>

#include "stringlib.h"
#include "hg_sampler.h"
#include "sentence_metadata.h"
#include "ns.h"
#include "ns_docscorer.h"
#include "verbose.h"
#include "viterbi.h"
#include "hg.h"
#include "prob.h"
#include "kbest.h"
#include "ff_register.h"
#include "decoder.h"
#include "filelib.h"
#include "fdict.h"
#include "weights.h"
#include "sparse_vector.h"
#include "sampler.h"

using namespace std;
namespace po = boost::program_options;

bool invert_score;
std::tr1::shared_ptr<MT19937> rng;

void RandomPermutation(int len, vector<int>* p_ids) {
  vector<int>& ids = *p_ids;
  ids.resize(len);
  for (int i = 0; i < len; ++i) ids[i] = i;
  for (int i = len; i > 0; --i) {
    int j = rng->next() * i;
    if (j == i) i--;
    swap(ids[i-1], ids[j]);
  }
}

bool InitCommandLine(int argc, char** argv, po::variables_map* conf) {
  po::options_description opts("Configuration options");
  opts.add_options()
        ("input_weights,w",po::value<string>(),"Input feature weights file")
        ("source,i",po::value<string>(),"Source file for development set")
        ("passes,p", po::value<int>()->default_value(15), "Number of passes through the training data")
        ("reference,r",po::value<vector<string> >(), "[REQD] Reference translation(s) (tokenized text file)")
        ("mt_metric,m",po::value<string>()->default_value("ibm_bleu"), "Scoring metric (ibm_bleu, nist_bleu, koehn_bleu, ter, combi)")
        ("max_step_size,C", po::value<double>()->default_value(0.01), "regularization strength (C)")
        ("mt_metric_scale,s", po::value<double>()->default_value(1.0), "Amount to scale MT loss function by")
        ("k_best_size,k", po::value<int>()->default_value(250), "Size of hypothesis list to search for oracles")
        ("sample_forest,f", "Instead of a k-best list, sample k hypotheses from the decoder's forest")
        ("sample_forest_unit_weight_vector,x", "Before sampling (must use -f option), rescale the weight vector used so it has unit length; this may improve the quality of the samples")
        ("random_seed,S", po::value<uint32_t>(), "Random seed (if not specified, /dev/random will be used)")
        ("decoder_config,c",po::value<string>(),"Decoder configuration file");
  po::options_description clo("Command line options");
  clo.add_options()
        ("config", po::value<string>(), "Configuration file")
        ("help,h", "Print this help message and exit");
  po::options_description dconfig_options, dcmdline_options;
  dconfig_options.add(opts);
  dcmdline_options.add(opts).add(clo);

  po::store(parse_command_line(argc, argv, dcmdline_options), *conf);
  if (conf->count("config")) {
    ifstream config((*conf)["config"].as<string>().c_str());
    po::store(po::parse_config_file(config, dconfig_options), *conf);
  }
  po::notify(*conf);

  if (conf->count("help") || !conf->count("input_weights") || !conf->count("source") || !conf->count("decoder_config") || !conf->count("reference")) {
    cerr << dcmdline_options << endl;
    return false;
  }
  return true;
}

static const double kMINUS_EPSILON = -1e-6;

struct HypothesisInfo {
  SparseVector<double> features;
  double mt_metric;
};

struct GoodBadOracle {
  std::tr1::shared_ptr<HypothesisInfo> good;
  std::tr1::shared_ptr<HypothesisInfo> bad;
};

struct TrainingObserver : public DecoderObserver {
  TrainingObserver(const int k, const DocumentScorer& d, const EvaluationMetric& m, bool sf, vector<GoodBadOracle>* o) : ds(d), metric(m), oracles(*o), kbest_size(k), sample_forest(sf) {}
  const DocumentScorer& ds;
  const EvaluationMetric& metric;
  vector<GoodBadOracle>& oracles;
  std::tr1::shared_ptr<HypothesisInfo> cur_best;
  const int kbest_size;
  const bool sample_forest;

  const HypothesisInfo& GetCurrentBestHypothesis() const {
    return *cur_best;
  }

  virtual void NotifyTranslationForest(const SentenceMetadata& smeta, Hypergraph* hg) {
    UpdateOracles(smeta.GetSentenceID(), *hg);
  }

  std::tr1::shared_ptr<HypothesisInfo> MakeHypothesisInfo(const SparseVector<double>& feats, const double score) {
    std::tr1::shared_ptr<HypothesisInfo> h(new HypothesisInfo);
    h->features = feats;
    h->mt_metric = score;
    return h;
  }

  void UpdateOracles(int sent_id, const Hypergraph& forest) {
    std::tr1::shared_ptr<HypothesisInfo>& cur_good = oracles[sent_id].good;
    std::tr1::shared_ptr<HypothesisInfo>& cur_bad = oracles[sent_id].bad;
    cur_bad.reset();  // TODO get rid of??

    if (sample_forest) {
      vector<WordID> cur_prediction;
      ViterbiESentence(forest, &cur_prediction);
      SufficientStats sstats;
      ds[sent_id]->Evaluate(cur_prediction, &sstats);
      float sentscore = metric.ComputeScore(sstats);
      cur_best = MakeHypothesisInfo(ViterbiFeatures(forest), sentscore);

      vector<HypergraphSampler::Hypothesis> samples;
      HypergraphSampler::sample_hypotheses(forest, kbest_size, &*rng, &samples);
      for (unsigned i = 0; i < samples.size(); ++i) {
        ds[sent_id]->Evaluate(samples[i].words, &sstats);
        float sentscore = metric.ComputeScore(sstats);
        if (invert_score) sentscore *= -1.0;
        if (!cur_good || sentscore > cur_good->mt_metric)
          cur_good = MakeHypothesisInfo(samples[i].fmap, sentscore);
        if (!cur_bad || sentscore < cur_bad->mt_metric)
          cur_bad = MakeHypothesisInfo(samples[i].fmap, sentscore);
      }
    } else {
      KBest::KBestDerivations<vector<WordID>, ESentenceTraversal> kbest(forest, kbest_size);
      SufficientStats sstats;
      for (int i = 0; i < kbest_size; ++i) {
        const KBest::KBestDerivations<vector<WordID>, ESentenceTraversal>::Derivation* d =
          kbest.LazyKthBest(forest.nodes_.size() - 1, i);
        if (!d) break;
        ds[sent_id]->Evaluate(d->yield, &sstats);
        float sentscore = metric.ComputeScore(sstats);
        if (invert_score) sentscore *= -1.0;
        // cerr << TD::GetString(d->yield) << " ||| " << d->score << " ||| " << sentscore << endl;
        if (i == 0)
          cur_best = MakeHypothesisInfo(d->feature_values, sentscore);
        if (!cur_good || sentscore > cur_good->mt_metric)
          cur_good = MakeHypothesisInfo(d->feature_values, sentscore);
        if (!cur_bad || sentscore < cur_bad->mt_metric)
          cur_bad = MakeHypothesisInfo(d->feature_values, sentscore);
      }
      //cerr << "GOOD: " << cur_good->mt_metric << endl;
      //cerr << " CUR: " << cur_best->mt_metric << endl;
      //cerr << " BAD: " << cur_bad->mt_metric << endl;
    }
  }
};

void ReadTrainingCorpus(const string& fname, vector<string>* c) {
  ReadFile rf(fname);
  istream& in = *rf.stream();
  string line;
  while(in) {
    getline(in, line);
    if (!in) break;
    c->push_back(line);
  }
}

bool ApproxEqual(double a, double b) {
  if (a == b) return true;
  return (fabs(a-b)/fabs(b)) < 0.000001;
}

int main(int argc, char** argv) {
  register_feature_functions();
  SetSilent(true);  // turn off verbose decoder output

  po::variables_map conf;
  if (!InitCommandLine(argc, argv, &conf)) return 1;

  if (conf.count("random_seed"))
    rng.reset(new MT19937(conf["random_seed"].as<uint32_t>()));
  else
    rng.reset(new MT19937);
  const bool sample_forest = conf.count("sample_forest") > 0;
  const bool sample_forest_unit_weight_vector = conf.count("sample_forest_unit_weight_vector") > 0;
  if (sample_forest_unit_weight_vector && !sample_forest) {
    cerr << "Cannot --sample_forest_unit_weight_vector without --sample_forest" << endl;
    return 1;
  }
  vector<string> corpus;
  ReadTrainingCorpus(conf["source"].as<string>(), &corpus);

  string metric_name = UppercaseString(conf["mt_metric"].as<string>());
  if (metric_name == "COMBI") {
    cerr << "WARNING: 'combi' metric is no longer supported, switching to 'COMB:TER=-0.5;IBM_BLEU=0.5'\n";
    metric_name = "COMB:TER=-0.5;IBM_BLEU=0.5";
  } else if (metric_name == "BLEU") {
    cerr << "WARNING: 'BLEU' is ambiguous, assuming 'IBM_BLEU'\n";
    metric_name = "IBM_BLEU";
  }
  EvaluationMetric* metric = EvaluationMetric::Instance(metric_name);
  DocumentScorer ds(metric, conf["reference"].as<vector<string> >());
  cerr << "Loaded " << ds.size() << " references for scoring with " << metric_name << endl;
  invert_score = metric->IsErrorMetric();

  if (ds.size() != corpus.size()) {
    cerr << "Mismatched number of references (" << ds.size() << ") and sources (" << corpus.size() << ")\n";
    return 1;
  }

  ReadFile ini_rf(conf["decoder_config"].as<string>());
  Decoder decoder(ini_rf.stream());

  // load initial weights
  vector<weight_t>& dense_weights = decoder.CurrentWeightVector();
  SparseVector<weight_t> lambdas;
  Weights::InitFromFile(conf["input_weights"].as<string>(), &dense_weights);
  Weights::InitSparseVector(dense_weights, &lambdas);

  const double max_step_size = conf["max_step_size"].as<double>();
  const double mt_metric_scale = conf["mt_metric_scale"].as<double>();

  assert(corpus.size() > 0);
  vector<GoodBadOracle> oracles(corpus.size());

  TrainingObserver observer(conf["k_best_size"].as<int>(), ds, *metric, sample_forest, &oracles);
  int cur_sent = 0;
  int lcount = 0;
  int normalizer = 0;
  double tot_loss = 0;
  int dots = 0;
  int cur_pass = 0;
  SparseVector<double> tot;
  tot += lambdas;          // initial weights
  normalizer++;            // count for initial weights
  int max_iteration = conf["passes"].as<int>() * corpus.size();
  string msg = "# MIRA tuned weights";
  string msga = "# MIRA tuned weights AVERAGED";
  vector<int> order;
  RandomPermutation(corpus.size(), &order);
  while (lcount <= max_iteration) {
    lambdas.init_vector(&dense_weights);
    if ((cur_sent * 40 / corpus.size()) > dots) { ++dots; cerr << '.'; }
    if (corpus.size() == cur_sent) {
      cerr << " [AVG METRIC LAST PASS=" << (tot_loss / corpus.size()) << "]\n";
      Weights::ShowLargestFeatures(dense_weights);
      cur_sent = 0;
      tot_loss = 0;
      dots = 0;
      ostringstream os;
      os << "weights.mira-pass" << (cur_pass < 10 ? "0" : "") << cur_pass << ".gz";
      SparseVector<double> x = tot;
      x /= normalizer;
      ostringstream sa;
      sa << "weights.mira-pass" << (cur_pass < 10 ? "0" : "") << cur_pass << "-avg.gz";
      x.init_vector(&dense_weights);
      Weights::WriteToFile(os.str(), dense_weights, true, &msg);
      ++cur_pass;
      RandomPermutation(corpus.size(), &order);
    }
    if (cur_sent == 0) {
      cerr << "PASS " << (lcount / corpus.size() + 1) << endl;
    }
    decoder.SetId(order[cur_sent]);
    double sc = 1.0;
    if (sample_forest_unit_weight_vector) {
      sc = lambdas.l2norm();
      if (sc > 0) {
        for (unsigned i = 0; i < dense_weights.size(); ++i)
          dense_weights[i] /= sc;
      }
    }
    decoder.Decode(corpus[order[cur_sent]], &observer);  // update oracles
    if (sc && sc != 1.0) {
      for (unsigned i = 0; i < dense_weights.size(); ++i)
        dense_weights[i] *= sc;
    }
    const HypothesisInfo& cur_hyp = observer.GetCurrentBestHypothesis();
    const HypothesisInfo& cur_good = *oracles[order[cur_sent]].good;
    const HypothesisInfo& cur_bad = *oracles[order[cur_sent]].bad;
    tot_loss += cur_hyp.mt_metric;
    if (!ApproxEqual(cur_hyp.mt_metric, cur_good.mt_metric)) {
      const double loss = cur_bad.features.dot(dense_weights) - cur_good.features.dot(dense_weights) +
          mt_metric_scale * (cur_good.mt_metric - cur_bad.mt_metric);
      //cerr << "LOSS: " << loss << endl;
      if (loss > 0.0) {
        SparseVector<double> diff = cur_good.features;
        diff -= cur_bad.features;
        double step_size = loss / diff.l2norm_sq();
        //cerr << loss << " " << step_size << " " << diff << endl;
        if (step_size > max_step_size) step_size = max_step_size;
        lambdas += (cur_good.features * step_size);
        lambdas -= (cur_bad.features * step_size);
        //cerr << "L: " << lambdas << endl;
      }
    }
    tot += lambdas;
    ++normalizer;
    ++lcount;
    ++cur_sent;
  }
  cerr << endl;
  Weights::WriteToFile("weights.mira-final.gz", dense_weights, true, &msg);
  tot /= normalizer;
  tot.init_vector(dense_weights);
  msg = "# MIRA tuned weights (averaged vector)";
  Weights::WriteToFile("weights.mira-final-avg.gz", dense_weights, true, &msg);
  cerr << "Optimization complete.\nAVERAGED WEIGHTS: weights.mira-final-avg.gz\n";
  return 0;
}

<|endoftext|>


<commit_before>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <vector>

#include "log/log_sink_file_backend.h"
#include "log/log_stacktrace.h"
#include "log/log_wrapper.h"
#include "std/thread.h"

#include "common/file_system.h"
#include "time/time_utility.h"

#include "algorithm/hash.h"
#include "random/random_generator.h"
#include "string/tquerystring.h"

//=======================================================================================================
void log_sample_func1(int times) {
    if (times > 0) {
        log_sample_func1(times - 1);
        return;
    }

    puts("");
    puts("===============begin log sample==============");

    if (util::log::is_stacktrace_enabled()) {
        puts("----------------test stacktrace begin--------------");
        char buffer[2048] = {0};
        util::log::stacktrace_write(buffer, sizeof(buffer));
        puts(buffer);
        puts("----------------test stacktrace end--------------");
    }

    WLOG_INIT(util::log::log_wrapper::categorize_t::DEFAULT, util::log::log_wrapper::level_t::LOG_LW_DEBUG);
    WLOG_GETCAT(util::log::log_wrapper::categorize_t::DEFAULT)->set_stacktrace_level(util::log::log_wrapper::level_t::LOG_LW_INFO);

    WLOG_GETCAT(util::log::log_wrapper::categorize_t::DEFAULT)->clear_sinks();

    PSTDERROR("try to print error log.\n");
    PSTDOK("try to print ok log.\n");

    WLOGNOTICE("notice log %d", 0);

    util::log::log_sink_file_backend filed_backend;
    filed_backend.set_max_file_size(256);
    filed_backend.set_rotate_size(3);
    filed_backend.set_file_pattern("%Y-%m-%d/%S/%N.log");

    WLOG_GETCAT(util::log::log_wrapper::categorize_t::DEFAULT)->add_sink(filed_backend);

    for (int i = 0; i < 16; ++i) {
        WLOGDEBUG("first dir test log: %d", i);
    }

    THREAD_SLEEP_MS(1000);
    util::time::time_utility::update();

    for (int i = 0; i < 16; ++i) {
        WLOGDEBUG("second dir log: %d", i);
    }

    unsigned long long ull_test_in_mingw = 64;
    WLOGINFO("%llu", ull_test_in_mingw);

    WLOG_GETCAT(util::log::log_wrapper::categorize_t::DEFAULT)
        ->set_sink(WLOG_GETCAT(util::log::log_wrapper::categorize_t::DEFAULT)->sink_size() - 1,
                   util::log::log_wrapper::level_t::LOG_LW_DEBUG, util::log::log_wrapper::level_t::LOG_LW_DEBUG);
    WLOGDEBUG("Debug still available %llu", ull_test_in_mingw);
    WLOGINFO("Info not available now %llu", ull_test_in_mingw);

    printf("log are located at %s\n", util::file_system::get_cwd().c_str());
    puts("===============end log sample==============");

    WLOG_GETCAT(util::log::log_wrapper::categorize_t::DEFAULT)->pop_sink();
    WLOGERROR("No log sink now");
}

class log_sample_functor2 {
public:
    void func2(int times) {
        if (times & 0x01) {
            func2(times - 1);
        } else {
            log_sample_func1(times - 1);
        }
    }
};

class log_sample_functor3 {
public:
    static void func3(int times) {
        if (times & 0x01) {
            func3(times - 1);
        } else {
            log_sample_functor2 f;
            f.func2(times - 1);
        }
    }
};

struct log_sample_functor4 {
    void operator()(int times) {
        if (times & 0x01) {
            (*this)(times - 1);
        } else {
            log_sample_functor3::func3(times - 1);
        }
    }
};

static void log_sample_func5(int times) {
    if (times & 0x01) {
        log_sample_func5(times - 1);
    } else {
        log_sample_functor4 f;
        f(times - 1);
    }
}

void log_sample() { log_sample_func5(9); }

//=======================================================================================================

//=======================================================================================================
void random_sample() {
    printf("\n");
    printf("===============begin random sample==============\n");
    util::random::mt19937 gen1;
    gen1.init_seed(123);

    printf("Random - mt19937: %u\n", gen1.random());
    printf("Random - mt19937: %u\n", gen1());
    printf("Random - mt19937 - between [100, 10000): %d\n", gen1.random_between(100, 10000));

    printf("===============end random sample==============\n");
}

//=======================================================================================================

//=======================================================================================================
void tquerystring_sample() {
    printf("\n");
    printf("===============begin querystring sample==============\n");

    util::tquerystring encode, decode;

    encode.set("a", "wulala");
    encode.set("page", "ok!");

    std::string output;
    encode.encode(output);

    util::types::item_array::ptr_type arr = encode.create_array();

    arr->append("blablabla...");
    arr->append("a and b is ab");
    util::types::item_object::ptr_type obj = encode.create_object();
    obj->set("so", "");
    arr->append(obj);

    encode.set("c", arr);

    std::cout << "encode => " << encode.to_string() << std::endl;
    std::cout << "encode (old) => " << output << std::endl;
    std::cout << "Array => " << arr->to_string() << std::endl;

    decode.decode(encode.to_string().c_str());
    std::cout << "decode => " << decode.to_string() << std::endl;

    printf("===============end querystring sample================\n");
}

//=======================================================================================================

//=======================================================================================================
void hash_sample() {
    std::cout << std::endl << "===============begin hash sample=============="<< std::endl;
    char str_buff[] = "Hello World!\nI'm OWenT\n";
    std::cout << "Hashed String: " << std::endl << str_buff << std::endl;
    std::cout << "FNV-1:   " << util::hash::hash_fnv1<uint32_t>(str_buff, strlen(str_buff)) << std::endl;
    std::cout << "FNV-1A:  " << util::hash::hash_fnv1a<uint32_t>(str_buff, strlen(str_buff)) << std::endl;
    std::cout << "SDBM:    " << util::hash::hash_sdbm<uint32_t>(str_buff, strlen(str_buff)) << std::endl;
    std::cout << "RS:      " << util::hash::hash_rs<uint32_t>(str_buff, strlen(str_buff)) << std::endl;
    std::cout << "JS:      " << util::hash::hash_js<uint32_t>(str_buff, strlen(str_buff)) << std::endl;
    std::cout << "PJW:     " << util::hash::hash_pjw<uint32_t>(str_buff, strlen(str_buff)) << std::endl;
    std::cout << "ELF:     " << util::hash::hash_elf<uint32_t>(str_buff, strlen(str_buff)) << std::endl;
    std::cout << "BKDR:    " << util::hash::hash_bkdr<uint32_t>(str_buff, strlen(str_buff)) << std::endl;
    std::cout << "DJB:     " << util::hash::hash_djb<uint32_t>(str_buff, strlen(str_buff)) << std::endl;
    std::cout << "AP:      " << util::hash::hash_ap<uint32_t>(str_buff, strlen(str_buff)) << std::endl;
    std::cout << "===============end hash sample==============" << std::endl;
}
//=======================================================================================================

//=======================================================================================================
extern int cmd_option_sample_main();

void cmd_option_sample() {
    puts("");
    puts("===============begin cmd_option sample==============");
    cmd_option_sample_main();
    puts("===============end cmd_option sample==============");
}
//=======================================================================================================

int main(int argc, char **argv) {
    util::time::time_utility::update();

    tquerystring_sample();
    hash_sample();
    random_sample();
    log_sample();
    cmd_option_sample();
    return 0;
}
<commit_msg>test msvc12<commit_after>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <vector>

#include "log/log_sink_file_backend.h"
#include "log/log_stacktrace.h"
#include "log/log_wrapper.h"
#include "std/thread.h"

#include "common/file_system.h"
#include "time/time_utility.h"

#include "algorithm/hash.h"
#include "random/random_generator.h"
#include "string/tquerystring.h"

#define CALL_SAMPLE(x) \
    std::cout << std::endl << "=============== begin " << #x << " =============="<<std::endl; \
    x();                                                                                \
    std::cout << "===============  end " << #x << "  ==============" << std::endl

//=======================================================================================================
void log_sample_func1(int times) {
    if (times > 0) {
        log_sample_func1(times - 1);
        return;
    }

    if (util::log::is_stacktrace_enabled()) {
        std::cout << "----------------test stacktrace begin--------------" << std::endl;
        char buffer[2048] = {0};
        util::log::stacktrace_write(buffer, sizeof(buffer));
        std::cout << buffer << std::endl;
        std::cout << "----------------test stacktrace end--------------" << std::endl;
    }

    WLOG_INIT(util::log::log_wrapper::categorize_t::DEFAULT, util::log::log_wrapper::level_t::LOG_LW_DEBUG);
    WLOG_GETCAT(util::log::log_wrapper::categorize_t::DEFAULT)->set_stacktrace_level(util::log::log_wrapper::level_t::LOG_LW_INFO);

    WLOG_GETCAT(util::log::log_wrapper::categorize_t::DEFAULT)->clear_sinks();

    std::cout << "----------------setup log_wrapper done--------------" << std::endl;

    PSTDERROR("try to print error log.\n");
    PSTDOK("try to print ok log.\n");

    std::cout << "----------------sample for PSTDOK/PSTDERROR done--------------" << std::endl;

    WLOGNOTICE("notice log %d", 0);

    util::log::log_sink_file_backend filed_backend;
    filed_backend.set_max_file_size(256);
    filed_backend.set_rotate_size(3);
    filed_backend.set_file_pattern("%Y-%m-%d/%S/%N.log");

    WLOG_GETCAT(util::log::log_wrapper::categorize_t::DEFAULT)->add_sink(filed_backend);
    std::cout << "----------------setup file system log sink done--------------" << std::endl;

    for (int i = 0; i < 16; ++i) {
        WLOGDEBUG("first dir test log: %d", i);
    }

    THREAD_SLEEP_MS(1000);
    util::time::time_utility::update();

    for (int i = 0; i < 16; ++i) {
        WLOGDEBUG("second dir log: %d", i);
    }

    unsigned long long ull_test_in_mingw = 64;
    WLOGINFO("%llu", ull_test_in_mingw);

    WLOG_GETCAT(util::log::log_wrapper::categorize_t::DEFAULT)
        ->set_sink(WLOG_GETCAT(util::log::log_wrapper::categorize_t::DEFAULT)->sink_size() - 1,
                   util::log::log_wrapper::level_t::LOG_LW_DEBUG, util::log::log_wrapper::level_t::LOG_LW_DEBUG);
    WLOGDEBUG("Debug still available %llu", ull_test_in_mingw);
    WLOGINFO("Info not available now %llu", ull_test_in_mingw);

    std::cout<< "log are located at "<< util::file_system::get_cwd().c_str()<< std::endl;

    WLOG_GETCAT(util::log::log_wrapper::categorize_t::DEFAULT)->pop_sink();
    WLOGERROR("No log sink now");
}

class log_sample_functor2 {
public:
    void func2(int times) {
        if (times & 0x01) {
            func2(times - 1);
        } else {
            log_sample_func1(times - 1);
        }
    }
};

class log_sample_functor3 {
public:
    static void func3(int times) {
        if (times & 0x01) {
            func3(times - 1);
        } else {
            log_sample_functor2 f;
            f.func2(times - 1);
        }
    }
};

struct log_sample_functor4 {
    void operator()(int times) {
        if (times & 0x01) {
            (*this)(times - 1);
        } else {
            log_sample_functor3::func3(times - 1);
        }
    }
};

static void log_sample_func5(int times) {
    if (times & 0x01) {
        log_sample_func5(times - 1);
    } else {
        log_sample_functor4 f;
        f(times - 1);
    }
}

void log_sample() { log_sample_func5(9); }

//=======================================================================================================

//=======================================================================================================
void random_sample() {
    util::random::mt19937 gen1;
    gen1.init_seed(123);

    std::cout << "Random - mt19937: %u" << gen1.random() << std::endl;
    std::cout << "Random - mt19937: %u" << gen1() << std::endl;
    std::cout << "Random - mt19937 - between [100, 10000): %d" << gen1.random_between(100, 10000) << std::endl;
}

//=======================================================================================================

//=======================================================================================================
void tquerystring_sample() {
    util::tquerystring encode, decode;

    encode.set("a", "wulala");
    encode.set("page", "ok!");

    std::string output;
    encode.encode(output);

    util::types::item_array::ptr_type arr = encode.create_array();

    arr->append("blablabla...");
    arr->append("a and b is ab");
    util::types::item_object::ptr_type obj = encode.create_object();
    obj->set("so", "");
    arr->append(obj);

    encode.set("c", arr);

    std::cout << "encode => " << encode.to_string() << std::endl;
    std::cout << "encode (old) => " << output << std::endl;
    std::cout << "Array => " << arr->to_string() << std::endl;

    decode.decode(encode.to_string().c_str());
    std::cout << "decode => " << decode.to_string() << std::endl;
}

//=======================================================================================================

//=======================================================================================================
void hash_sample() {
    char str_buff[] = "Hello World!\nI'm OWenT\n";
    std::cout << "Hashed String: " << std::endl << str_buff << std::endl;
    std::cout << "FNV-1:   " << util::hash::hash_fnv1<uint32_t>(str_buff, strlen(str_buff)) << std::endl;
    std::cout << "FNV-1A:  " << util::hash::hash_fnv1a<uint32_t>(str_buff, strlen(str_buff)) << std::endl;
    std::cout << "SDBM:    " << util::hash::hash_sdbm<uint32_t>(str_buff, strlen(str_buff)) << std::endl;
    std::cout << "RS:      " << util::hash::hash_rs<uint32_t>(str_buff, strlen(str_buff)) << std::endl;
    std::cout << "JS:      " << util::hash::hash_js<uint32_t>(str_buff, strlen(str_buff)) << std::endl;
    std::cout << "PJW:     " << util::hash::hash_pjw<uint32_t>(str_buff, strlen(str_buff)) << std::endl;
    std::cout << "ELF:     " << util::hash::hash_elf<uint32_t>(str_buff, strlen(str_buff)) << std::endl;
    std::cout << "BKDR:    " << util::hash::hash_bkdr<uint32_t>(str_buff, strlen(str_buff)) << std::endl;
    std::cout << "DJB:     " << util::hash::hash_djb<uint32_t>(str_buff, strlen(str_buff)) << std::endl;
    std::cout << "AP:      " << util::hash::hash_ap<uint32_t>(str_buff, strlen(str_buff)) << std::endl;
}
//=======================================================================================================

//=======================================================================================================
extern int cmd_option_sample_main();

void cmd_option_sample() {
    cmd_option_sample_main();
}
//=======================================================================================================

int main(int argc, char **argv) {
    util::time::time_utility::update();

    CALL_SAMPLE(tquerystring_sample);
    CALL_SAMPLE(hash_sample);
    CALL_SAMPLE(random_sample);
    CALL_SAMPLE(log_sample);
    CALL_SAMPLE(cmd_option_sample);

    return 0;
}
<|endoftext|>


<commit_before><commit_msg>DeleteRange is only called from within ScColumn, and should be private.<commit_after><|endoftext|>


<commit_before>// Copyright (c) 2019, 2020 by Robert Bosch GmbH, Apex.AI Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "iceoryx_posh/internal/log/posh_logging.hpp"
#include "iceoryx_posh/internal/runtime/ipc_interface_base.hpp"
#include "iceoryx_posh/internal/runtime/ipc_message.hpp"
#include "iceoryx_utils/cxx/convert.hpp"
#include "iceoryx_utils/cxx/smart_c.hpp"
#include "iceoryx_utils/error_handling/error_handling.hpp"
#include "iceoryx_utils/internal/posix_wrapper/message_queue.hpp"
#include "iceoryx_utils/internal/posix_wrapper/timespec.hpp"

#include <thread>

namespace iox
{
namespace runtime
{
IpcMessageType stringToIpcMessageType(const char* str) noexcept
{
    std::underlying_type<IpcMessageType>::type msg;
    bool noError = cxx::convert::stringIsNumber(str, cxx::convert::NumberType::INTEGER);
    noError &= noError ? (cxx::convert::fromString(str, msg)) : false;
    noError &= noError ? !(static_cast<std::underlying_type<IpcMessageType>::type>(IpcMessageType::BEGIN) >= msg
                           || static_cast<std::underlying_type<IpcMessageType>::type>(IpcMessageType::END) <= msg)
                       : false;
    return noError ? (static_cast<IpcMessageType>(msg)) : IpcMessageType::NOTYPE;
}

std::string IpcMessageTypeToString(const IpcMessageType msg) noexcept
{
    return std::to_string(static_cast<std::underlying_type<IpcMessageType>::type>(msg));
}

IpcMessageErrorType stringToIpcMessageErrorType(const char* str) noexcept
{
    std::underlying_type<IpcMessageErrorType>::type msg;
    bool noError = cxx::convert::stringIsNumber(str, cxx::convert::NumberType::INTEGER);
    noError &= noError ? (cxx::convert::fromString(str, msg)) : false;
    noError &= noError
                   ? !(static_cast<std::underlying_type<IpcMessageErrorType>::type>(IpcMessageErrorType::BEGIN) >= msg
                       || static_cast<std::underlying_type<IpcMessageErrorType>::type>(IpcMessageErrorType::END) <= msg)
                   : false;
    return noError ? (static_cast<IpcMessageErrorType>(msg)) : IpcMessageErrorType::NOTYPE;
}

std::string IpcMessageErrorTypeToString(const IpcMessageErrorType msg) noexcept
{
    return std::to_string(static_cast<std::underlying_type<IpcMessageErrorType>::type>(msg));
}

IpcInterfaceBase::IpcInterfaceBase(const ProcessName_t& InterfaceName,
                                   const uint64_t maxMessages,
                                   const uint64_t messageSize) noexcept
    : m_interfaceName(InterfaceName)
{
    m_maxMessages = maxMessages;
    m_maxMessageSize = messageSize;
    if (m_maxMessageSize > posix::MessageQueue::MAX_MESSAGE_SIZE)
    {
        LogWarn() << "Message size too large, reducing from " << messageSize << " to "
                  << posix::MessageQueue::MAX_MESSAGE_SIZE;
        m_maxMessageSize = posix::MessageQueue::MAX_MESSAGE_SIZE;
    }
}

bool IpcInterfaceBase::receive(IpcMessage& answer) const noexcept
{
    auto message = m_ipcChannel.receive();
    if (message.has_error())
    {
        return false;
    }

    return IpcInterfaceBase::setMessageFromString(message.value().c_str(), answer);
}

bool IpcInterfaceBase::timedReceive(const units::Duration timeout, IpcMessage& answer) const noexcept
{
    return !m_ipcChannel.timedReceive(timeout)
                .and_then([&answer](auto& message) { IpcInterfaceBase::setMessageFromString(message.c_str(), answer); })
                .has_error()
           && answer.isValid();
}

bool IpcInterfaceBase::setMessageFromString(const char* buffer, IpcMessage& answer) noexcept
{
    answer.setMessage(buffer);
    if (!answer.isValid())
    {
        LogError() << "The received message " << answer.getMessage() << " is not valid";
        return false;
    }
    return true;
}

bool IpcInterfaceBase::send(const IpcMessage& msg) const noexcept
{
    if (!msg.isValid())
    {
        LogError() << "Trying to send the message " << msg.getMessage() << " which "
                   << "does not follow the specified syntax.";
        return false;
    }

    auto logLengthError = [&msg](posix::IpcChannelError& error) {
        if (error == posix::IpcChannelError::MESSAGE_TOO_LONG)
        {
            const size_t messageSize =
                static_cast<size_t>(msg.getMessage().size()) + posix::MessageQueue::NULL_TERMINATOR_SIZE;
            LogError() << "msg size of " << messageSize << "bigger than configured max message size";
        }
    };
    return !m_ipcChannel.send(msg.getMessage()).or_else(logLengthError).has_error();
}

bool IpcInterfaceBase::timedSend(const IpcMessage& msg, units::Duration timeout) const noexcept
{
    if (!msg.isValid())
    {
        LogError() << "Trying to send the message " << msg.getMessage() << " which "
                   << "does not follow the specified syntax.";
        return false;
    }

    auto logLengthError = [&msg](posix::IpcChannelError& error) {
        if (error == posix::IpcChannelError::MESSAGE_TOO_LONG)
        {
            const size_t messageSize =
                static_cast<size_t>(msg.getMessage().size()) + posix::MessageQueue::NULL_TERMINATOR_SIZE;
            LogError() << "msg size of " << messageSize << "bigger than configured max message size";
        }
    };
    return !m_ipcChannel.timedSend(msg.getMessage(), timeout).or_else(logLengthError).has_error();
}

const ProcessName_t& IpcInterfaceBase::getInterfaceName() const noexcept
{
    return m_interfaceName;
}

bool IpcInterfaceBase::isInitialized() const noexcept
{
    return m_ipcChannel.isInitialized();
}

bool IpcInterfaceBase::openMessageQueue(const posix::IpcChannelSide channelSide) noexcept
{
    m_ipcChannel.destroy();

    m_channelSide = channelSide;
    IpcChannelType::create(
        m_interfaceName, posix::IpcChannelMode::BLOCKING, m_channelSide, m_maxMessageSize, m_maxMessages)
        .and_then([this](auto& ipcChannel) { this->m_ipcChannel = std::move(ipcChannel); });

    return m_ipcChannel.isInitialized();
}

bool IpcInterfaceBase::closeMessageQueue() noexcept
{
    return !m_ipcChannel.destroy().has_error();
}

bool IpcInterfaceBase::reopen() noexcept
{
    return openMessageQueue(m_channelSide);
}

bool IpcInterfaceBase::mqMapsToFile() noexcept
{
    return !m_ipcChannel.isOutdated().value_or(true);
}

bool IpcInterfaceBase::hasClosableMessageQueue() const noexcept
{
    return m_ipcChannel.isInitialized();
}

void IpcInterfaceBase::cleanupOutdatedMessageQueue(const ProcessName_t& name) noexcept
{
    if (posix::MessageQueue::unlinkIfExists(name).value_or(false))
    {
        LogWarn() << "IPC channel still there, doing an unlink of " << name;
    }
}

} // namespace runtime
} // namespace iox
<commit_msg>iox-#381 Remove duplicate file<commit_after><|endoftext|>