<commit_before>/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "paddle/fluid/memory/detail/buddy_allocator.h"

#include <memory>

#include "gflags/gflags.h"
#include "gtest/gtest.h"
#include "paddle/fluid/memory/detail/system_allocator.h"
#include "paddle/fluid/platform/gpu_info.h"

#ifdef PADDLE_WITH_CUDA
#include <cuda_runtime.h>

DECLARE_double(fraction_of_gpu_memory_to_use);
DECLARE_uint64(initial_gpu_memory_in_mb);
DECLARE_uint64(reallocate_gpu_memory_in_mb);
#endif

namespace paddle {
namespace memory {
namespace detail {

constexpr static int TEST_GPU_ID = 0;

int* TestBuddyAllocator(BuddyAllocator* allocator, size_t size_bytes,
                        bool use_system_allocator = false,
                        bool free_ptr = true) {
  bool freed = false;
  size_t used_bytes = allocator->Used();

  if (size_bytes > 0) {
    void* p = allocator->Alloc(size_bytes);

    EXPECT_NE(p, nullptr);

#ifdef PADDLE_WITH_CUDA
    if (size_bytes < allocator->GetMaxChunkSize()) {
#else
    if (size_bytes < allocator->GetMaxChunkSize()) {
#endif
      // Not allocate from SystemAllocator
      EXPECT_FALSE(use_system_allocator);
      EXPECT_GE(allocator->Used(), used_bytes + size_bytes);
    } else {
      // Allocate from SystemAllocator doesn't count in Used()
      EXPECT_TRUE(use_system_allocator);
      EXPECT_EQ(allocator->Used(), used_bytes);
    }

    int* intp = static_cast<int*>(p);
    if (!free_ptr) {
      return intp;
    }
    std::shared_ptr<int> ptr(intp, [&](void* p) {
      allocator->Free(intp);
      freed = true;
    });
  } else {
    freed = true;
  }

  EXPECT_EQ(used_bytes, allocator->Used());
  EXPECT_TRUE(freed);
  return nullptr;
}

#ifdef PADDLE_WITH_CUDA
TEST(BuddyAllocator, GpuFraction) {
  // In a 16 GB machine, the pool size will be about 160 MB
  FLAGS_fraction_of_gpu_memory_to_use = 0.01;
  FLAGS_initial_gpu_memory_in_mb = 0;
  FLAGS_reallocate_gpu_memory_in_mb = 0;

  BuddyAllocator buddy_allocator(
      std::unique_ptr<SystemAllocator>(new GPUAllocator(TEST_GPU_ID)),
      platform::GpuMinChunkSize(), platform::GpuMaxChunkSize());

  // Less than pool size
  TestBuddyAllocator(&buddy_allocator, 10);
  TestBuddyAllocator(&buddy_allocator, 10 << 10);
  TestBuddyAllocator(&buddy_allocator, 10 << 20);

  // Greater than max chunk size
  TestBuddyAllocator(&buddy_allocator, 499 << 20,
                     /* use_system_allocator = */ true);
  TestBuddyAllocator(&buddy_allocator, 2 * static_cast<size_t>(1 << 30),
                     /* use_system_allocator = */ true);
}

TEST(BuddyAllocator, InitRealloc) {
  FLAGS_initial_gpu_memory_in_mb = 100;
  FLAGS_reallocate_gpu_memory_in_mb = 50;

  EXPECT_EQ(platform::GpuMaxChunkSize(), static_cast<size_t>(100 << 20));

  BuddyAllocator buddy_allocator(
      std::unique_ptr<SystemAllocator>(new GPUAllocator(TEST_GPU_ID)),
      platform::GpuMinChunkSize(), platform::GpuMaxChunkSize());

  // Less then initial size and reallocate size
  TestBuddyAllocator(&buddy_allocator, 10 << 20);
  // Between initial size and reallocate size and not exceed pool
  TestBuddyAllocator(&buddy_allocator, 80 << 20);
  TestBuddyAllocator(&buddy_allocator, 99 << 20);
  // Greater than max chunk size
  TestBuddyAllocator(&buddy_allocator, 101 << 20,
                     /* use_system_allocator = */ true);
  TestBuddyAllocator(&buddy_allocator, 2 * static_cast<size_t>(1 << 30),
                     /* use_system_allocator = */ true);
}

TEST(BuddyAllocator, ReallocSizeGreaterThanInit) {
  FLAGS_initial_gpu_memory_in_mb = 5;
  FLAGS_reallocate_gpu_memory_in_mb = 10;

  EXPECT_EQ(platform::GpuMaxChunkSize(), static_cast<size_t>(10 << 20));

  BuddyAllocator buddy_allocator(
      std::unique_ptr<SystemAllocator>(new GPUAllocator(TEST_GPU_ID)),
      platform::GpuMinChunkSize(), platform::GpuMaxChunkSize());

  // Less than initial size and reallocate size
  TestBuddyAllocator(&buddy_allocator, 1 << 20);
  // Between initial size and reallocate size and exceed pool
  TestBuddyAllocator(&buddy_allocator, 6 << 20);
  TestBuddyAllocator(&buddy_allocator, 8 << 20);
  TestBuddyAllocator(&buddy_allocator, 9 << 20);
  // Greater than max trunk size
  TestBuddyAllocator(&buddy_allocator, 11 << 20,
                     /* use_system_allocator = */ true);
  TestBuddyAllocator(&buddy_allocator, 2 * static_cast<size_t>(1 << 30),
                     /* use_system_allocator = */ true);
}

TEST(BuddyAllocator, FractionRefillPool) {
  FLAGS_fraction_of_gpu_memory_to_use = 0.6;
  FLAGS_initial_gpu_memory_in_mb = 0;
  FLAGS_reallocate_gpu_memory_in_mb = 0;

  size_t max_chunk_size = platform::GpuMaxChunkSize();
  BuddyAllocator buddy_allocator(
      std::unique_ptr<SystemAllocator>(new GPUAllocator(TEST_GPU_ID)),
      platform::GpuMinChunkSize(), max_chunk_size);

  // Less than pool size
  int* p0 = TestBuddyAllocator(&buddy_allocator, max_chunk_size - 1000,
                               /* use_system_allocator = */ false,
                               /* free_ptr = */ false);
  // Max chunk size should be same during allocation
  EXPECT_EQ(max_chunk_size, buddy_allocator.GetMaxChunkSize());

  size_t alloc =
      platform::GpuAvailableMemToAlloc() * FLAGS_fraction_of_gpu_memory_to_use;
  // Exceed pool trigger refilling size of fraction of avaiable gpu, and should
  // be able to alloc 60% of the remaining GPU
  int* p1 = TestBuddyAllocator(&buddy_allocator, alloc,
                               /* use_system_allocator = */ false,
                               /* free_ptr = */ false);
  // Max chunk size should be same during allocation
  EXPECT_EQ(max_chunk_size, buddy_allocator.GetMaxChunkSize());

  alloc =
      platform::GpuAvailableMemToAlloc() * FLAGS_fraction_of_gpu_memory_to_use;
  // Exceed pool trigger refilling size of fraction of avaiable gpu, and should
  // be able to alloc 60% of the remaining GPU
  TestBuddyAllocator(&buddy_allocator, alloc,
                     /* use_system_allocator = */ false);
  // Max chunk size should be same during allocation
  EXPECT_EQ(max_chunk_size, buddy_allocator.GetMaxChunkSize());

  buddy_allocator.Free(p0);
  buddy_allocator.Free(p1);
}

TEST(BuddyAllocator, AllocFromAvailable) {
  FLAGS_fraction_of_gpu_memory_to_use = 0.7;
  FLAGS_initial_gpu_memory_in_mb = 0;
  FLAGS_reallocate_gpu_memory_in_mb = 0;

  size_t total = 0, available = 0;
  platform::SetDeviceId(TEST_GPU_ID);
  platform::GpuMemoryUsage(&available, &total);

  // Take half of available GPU
  void* p;
  cudaError_t result = cudaMalloc(&p, available >> 1);
  EXPECT_TRUE(result == cudaSuccess);

  // BuddyAllocator should be able to alloc the remaining GPU
  BuddyAllocator buddy_allocator(
      std::unique_ptr<SystemAllocator>(new GPUAllocator(TEST_GPU_ID)),
      platform::GpuMinChunkSize(), platform::GpuMaxChunkSize());

  TestBuddyAllocator(&buddy_allocator, 10);
  TestBuddyAllocator(&buddy_allocator, 10 << 10);
  TestBuddyAllocator(&buddy_allocator, 10 << 20);
  TestBuddyAllocator(&buddy_allocator, static_cast<size_t>(1 << 30));

  if (p) {
    EXPECT_TRUE(cudaFree(p) == cudaSuccess);
  }
}

TEST(BuddyAllocator, AllocFromAvailableWhenFractionIsOne) {
  FLAGS_fraction_of_gpu_memory_to_use = 1.0;
  FLAGS_initial_gpu_memory_in_mb = 0;
  FLAGS_reallocate_gpu_memory_in_mb = 0;

  void* p = nullptr;
  EXPECT_TRUE(cudaMalloc(&p, static_cast<size_t>(4) << 30) == cudaSuccess);

  // BuddyAllocator should be able to alloc the remaining GPU
  BuddyAllocator buddy_allocator(
      std::unique_ptr<SystemAllocator>(new GPUAllocator(TEST_GPU_ID)),
      platform::GpuMinChunkSize(), platform::GpuMaxChunkSize());

  TestBuddyAllocator(&buddy_allocator, static_cast<size_t>(1) << 30);
  TestBuddyAllocator(&buddy_allocator, static_cast<size_t>(5) << 30);

  if (p) {
    EXPECT_TRUE(cudaFree(p) == cudaSuccess);
  }
}

#endif

}  // namespace detail
}  // namespace memory
}  // namespace paddle
<commit_msg>fix buddy_allocator_test, test=develop (#19967)<commit_after>/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "paddle/fluid/memory/detail/buddy_allocator.h"

#include <memory>

#include "gflags/gflags.h"
#include "gtest/gtest.h"
#include "paddle/fluid/memory/detail/system_allocator.h"
#include "paddle/fluid/platform/gpu_info.h"

#ifdef PADDLE_WITH_CUDA
#include <cuda_runtime.h>

DECLARE_double(fraction_of_gpu_memory_to_use);
DECLARE_uint64(initial_gpu_memory_in_mb);
DECLARE_uint64(reallocate_gpu_memory_in_mb);
#endif

namespace paddle {
namespace memory {
namespace detail {

constexpr static int TEST_GPU_ID = 0;

int* TestBuddyAllocator(BuddyAllocator* allocator, size_t size_bytes,
                        bool use_system_allocator = false,
                        bool free_ptr = true) {
  bool freed = false;
  size_t used_bytes = allocator->Used();

  if (size_bytes > 0) {
    void* p = allocator->Alloc(size_bytes);

    EXPECT_NE(p, nullptr);

#ifdef PADDLE_WITH_CUDA
    if (size_bytes < allocator->GetMaxChunkSize()) {
#else
    if (size_bytes < allocator->GetMaxChunkSize()) {
#endif
      // Not allocate from SystemAllocator
      EXPECT_FALSE(use_system_allocator);
      EXPECT_GE(allocator->Used(), used_bytes + size_bytes);
    } else {
      // Allocate from SystemAllocator doesn't count in Used()
      EXPECT_TRUE(use_system_allocator);
      EXPECT_EQ(allocator->Used(), used_bytes);
    }

    int* intp = static_cast<int*>(p);
    if (!free_ptr) {
      return intp;
    }
    std::shared_ptr<int> ptr(intp, [&](void* p) {
      allocator->Free(intp);
      freed = true;
    });
  } else {
    freed = true;
  }

  EXPECT_EQ(used_bytes, allocator->Used());
  EXPECT_TRUE(freed);
  return nullptr;
}

#ifdef PADDLE_WITH_CUDA
TEST(BuddyAllocator, GpuFraction) {
  // In a 16 GB machine, the pool size will be about 160 MB
  FLAGS_fraction_of_gpu_memory_to_use = 0.01;
  FLAGS_initial_gpu_memory_in_mb = 0;
  FLAGS_reallocate_gpu_memory_in_mb = 0;

  BuddyAllocator buddy_allocator(
      std::unique_ptr<SystemAllocator>(new GPUAllocator(TEST_GPU_ID)),
      platform::GpuMinChunkSize(), platform::GpuMaxChunkSize());

  // Less than pool size
  TestBuddyAllocator(&buddy_allocator, 10);
  TestBuddyAllocator(&buddy_allocator, 10 << 10);
  TestBuddyAllocator(&buddy_allocator, 10 << 20);

  // Greater than max chunk size
  TestBuddyAllocator(&buddy_allocator, 499 << 20,
                     /* use_system_allocator = */ true);
  TestBuddyAllocator(&buddy_allocator, 2 * static_cast<size_t>(1 << 30),
                     /* use_system_allocator = */ true);
}

TEST(BuddyAllocator, InitRealloc) {
  FLAGS_initial_gpu_memory_in_mb = 100;
  FLAGS_reallocate_gpu_memory_in_mb = 50;

  EXPECT_EQ(platform::GpuMaxChunkSize(), static_cast<size_t>(100 << 20));

  BuddyAllocator buddy_allocator(
      std::unique_ptr<SystemAllocator>(new GPUAllocator(TEST_GPU_ID)),
      platform::GpuMinChunkSize(), platform::GpuMaxChunkSize());

  // Less then initial size and reallocate size
  TestBuddyAllocator(&buddy_allocator, 10 << 20);
  // Between initial size and reallocate size and not exceed pool
  TestBuddyAllocator(&buddy_allocator, 80 << 20);
  TestBuddyAllocator(&buddy_allocator, 99 << 20);
  // Greater than max chunk size
  TestBuddyAllocator(&buddy_allocator, 101 << 20,
                     /* use_system_allocator = */ true);
  TestBuddyAllocator(&buddy_allocator, 2 * static_cast<size_t>(1 << 30),
                     /* use_system_allocator = */ true);
}

TEST(BuddyAllocator, ReallocSizeGreaterThanInit) {
  FLAGS_initial_gpu_memory_in_mb = 5;
  FLAGS_reallocate_gpu_memory_in_mb = 10;

  EXPECT_EQ(platform::GpuMaxChunkSize(), static_cast<size_t>(10 << 20));

  BuddyAllocator buddy_allocator(
      std::unique_ptr<SystemAllocator>(new GPUAllocator(TEST_GPU_ID)),
      platform::GpuMinChunkSize(), platform::GpuMaxChunkSize());

  // Less than initial size and reallocate size
  TestBuddyAllocator(&buddy_allocator, 1 << 20);
  // Between initial size and reallocate size and exceed pool
  TestBuddyAllocator(&buddy_allocator, 6 << 20);
  TestBuddyAllocator(&buddy_allocator, 8 << 20);
  TestBuddyAllocator(&buddy_allocator, 9 << 20);
  // Greater than max trunk size
  TestBuddyAllocator(&buddy_allocator, 11 << 20,
                     /* use_system_allocator = */ true);
  TestBuddyAllocator(&buddy_allocator, 2 * static_cast<size_t>(1 << 30),
                     /* use_system_allocator = */ true);
}

TEST(BuddyAllocator, FractionRefillPool) {
  FLAGS_fraction_of_gpu_memory_to_use = 0.6;
  FLAGS_initial_gpu_memory_in_mb = 0;
  FLAGS_reallocate_gpu_memory_in_mb = 0;

  size_t max_chunk_size = platform::GpuMaxChunkSize();
  BuddyAllocator buddy_allocator(
      std::unique_ptr<SystemAllocator>(new GPUAllocator(TEST_GPU_ID)),
      platform::GpuMinChunkSize(), max_chunk_size);

  // Less than pool size
  int* p0 = TestBuddyAllocator(&buddy_allocator, max_chunk_size - 1000,
                               /* use_system_allocator = */ false,
                               /* free_ptr = */ false);
  // Max chunk size should be same during allocation
  EXPECT_EQ(max_chunk_size, buddy_allocator.GetMaxChunkSize());

  size_t alloc =
      platform::GpuAvailableMemToAlloc() * FLAGS_fraction_of_gpu_memory_to_use;
  // Exceed pool trigger refilling size of fraction of avaiable gpu, and should
  // be able to alloc 60% of the remaining GPU
  int* p1 = TestBuddyAllocator(&buddy_allocator, alloc,
                               /* use_system_allocator = */ false,
                               /* free_ptr = */ false);
  // Max chunk size should be same during allocation
  EXPECT_EQ(max_chunk_size, buddy_allocator.GetMaxChunkSize());

  alloc =
      platform::GpuAvailableMemToAlloc() * FLAGS_fraction_of_gpu_memory_to_use;
  // Exceed pool trigger refilling size of fraction of avaiable gpu, and should
  // be able to alloc 60% of the remaining GPU
  TestBuddyAllocator(&buddy_allocator, alloc,
                     /* use_system_allocator = */ false);
  // Max chunk size should be same during allocation
  EXPECT_EQ(max_chunk_size, buddy_allocator.GetMaxChunkSize());

  buddy_allocator.Free(p0);
  buddy_allocator.Free(p1);
}

TEST(BuddyAllocator, AllocFromAvailable) {
  FLAGS_fraction_of_gpu_memory_to_use = 0.7;
  FLAGS_initial_gpu_memory_in_mb = 0;
  FLAGS_reallocate_gpu_memory_in_mb = 0;

  size_t total = 0, available = 0;
  platform::SetDeviceId(TEST_GPU_ID);
  platform::GpuMemoryUsage(&available, &total);

  // Take half of available GPU
  void* p;
  cudaError_t result = cudaMalloc(&p, available >> 1);
  EXPECT_TRUE(result == cudaSuccess);

  // BuddyAllocator should be able to alloc the remaining GPU
  BuddyAllocator buddy_allocator(
      std::unique_ptr<SystemAllocator>(new GPUAllocator(TEST_GPU_ID)),
      platform::GpuMinChunkSize(), platform::GpuMaxChunkSize());

  TestBuddyAllocator(&buddy_allocator, 10);
  TestBuddyAllocator(&buddy_allocator, 10 << 10);
  TestBuddyAllocator(&buddy_allocator, 10 << 20);
  TestBuddyAllocator(&buddy_allocator, static_cast<size_t>(1 << 30));

  if (p) {
    EXPECT_TRUE(cudaFree(p) == cudaSuccess);
  }
}

TEST(BuddyAllocator, AllocFromAvailableWhenFractionIsOne) {
  FLAGS_fraction_of_gpu_memory_to_use = 1.0;
  FLAGS_initial_gpu_memory_in_mb = 0;
  FLAGS_reallocate_gpu_memory_in_mb = 0;

  void* p = nullptr;
  EXPECT_TRUE(cudaMalloc(&p, static_cast<size_t>(3) << 30) == cudaSuccess);

  // BuddyAllocator should be able to alloc the remaining GPU
  BuddyAllocator buddy_allocator(
      std::unique_ptr<SystemAllocator>(new GPUAllocator(TEST_GPU_ID)),
      platform::GpuMinChunkSize(), platform::GpuMaxChunkSize());

  TestBuddyAllocator(&buddy_allocator, static_cast<size_t>(1) << 30);
  TestBuddyAllocator(&buddy_allocator, static_cast<size_t>(2) << 30);

  if (p) {
    EXPECT_TRUE(cudaFree(p) == cudaSuccess);
  }
}

#endif

}  // namespace detail
}  // namespace memory
}  // namespace paddle
<|endoftext|>


<commit_before>/* Copyright 2017 Stanford University, NVIDIA Corporation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// clocks, timers for Realm

// nop, but helps IDEs
#include "timers.h"

#include "logging.h"

#ifdef __MACH__
#include <mach/clock.h>
#include <mach/mach.h>
#else
#include <time.h>
#endif

namespace Realm {

  ////////////////////////////////////////////////////////////////////////
  //
  // class Clock

  inline /*static*/ double Clock::current_time(bool absolute /*= false*/)
  {
#ifdef __MACH__
    mach_timespec_t ts;
    clock_serv_t cclock;
    host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock);
    clock_get_time(cclock, &ts);
    mach_port_deallocate(mach_task_self(), cclock);
#else
    struct timespec ts;
    clock_gettime(absolute ? CLOCK_REALTIME : CLOCK_MONOTONIC, &ts);
#endif
    double t = ts.tv_sec + (1e-9 * ts.tv_nsec);
    if(!absolute)
      t -= 1e-9 * zero_time;
    return t;
  }

  inline /*static*/ long long Clock::current_time_in_microseconds(bool absolute /*= false*/)
  {
#ifdef __MACH__
    mach_timespec_t ts;
    clock_serv_t cclock;
    host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock);
    clock_get_time(cclock, &ts);
    mach_port_deallocate(mach_task_self(), cclock);
#else
    struct timespec ts;
    clock_gettime(absolute ? CLOCK_REALTIME : CLOCK_MONOTONIC, &ts);
#endif
    long long t = (1000000LL * ts.tv_sec) + (ts.tv_nsec / 1000);
    if(!absolute)
      t -= zero_time / 1000;
    return t;
  }

  inline /*static*/ long long Clock::current_time_in_nanoseconds(bool absolute /*= false*/)
  {
#ifdef __MACH__
    mach_timespec_t ts;
    clock_serv_t cclock;
    host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock);
    clock_get_time(cclock, &ts);
    mach_port_deallocate(mach_task_self(), cclock);
#else
    struct timespec ts;
    clock_gettime(absolute ? CLOCK_REALTIME : CLOCK_MONOTONIC, &ts);
#endif
    long long t = (1000000000LL * ts.tv_sec) + ts.tv_nsec;
    if(!absolute)
      t -= zero_time;
    return t;
  }

  inline /*static*/ long long Clock::get_zero_time(void)
  {
    return zero_time;
  }

  inline /*static*/ void Clock::set_zero_time(void)
  {
    // this looks weird, but we can't use the absolute time because it uses
    //  a different system clock in POSIX-land, so we ask for the current
    //  relative time (based on whatever zero_time currently is) and add
    //  that in
    zero_time += current_time_in_nanoseconds(false);
  }

  ////////////////////////////////////////////////////////////////////////
  //
  // class Clock

  inline TimeStamp::TimeStamp(const char *_message, bool _difference, Logger *_logger /*= 0*/)
    : message(_message), difference(_difference), logger(_logger)
  {
    start_time = Clock::current_time();

    if(!difference) {
      if(logger)
	logger->print("%s %7.6f", message, start_time);
      else
	printf("%s %7.6f\n", message, start_time);
    }
  }

  inline TimeStamp::~TimeStamp(void)
  {
    if(difference) {
      double interval = Clock::current_time() - start_time;

      if(logger)
	logger->print("%s %7.6f", message, interval);
      else
	printf("%s %7.6f\n", message, interval);
    }
  }

}; // namespace Realm

<commit_msg>realm: use info instead of print for timers<commit_after>/* Copyright 2017 Stanford University, NVIDIA Corporation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// clocks, timers for Realm

// nop, but helps IDEs
#include "timers.h"

#include "logging.h"

#ifdef __MACH__
#include <mach/clock.h>
#include <mach/mach.h>
#else
#include <time.h>
#endif

namespace Realm {

  ////////////////////////////////////////////////////////////////////////
  //
  // class Clock

  inline /*static*/ double Clock::current_time(bool absolute /*= false*/)
  {
#ifdef __MACH__
    mach_timespec_t ts;
    clock_serv_t cclock;
    host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock);
    clock_get_time(cclock, &ts);
    mach_port_deallocate(mach_task_self(), cclock);
#else
    struct timespec ts;
    clock_gettime(absolute ? CLOCK_REALTIME : CLOCK_MONOTONIC, &ts);
#endif
    double t = ts.tv_sec + (1e-9 * ts.tv_nsec);
    if(!absolute)
      t -= 1e-9 * zero_time;
    return t;
  }

  inline /*static*/ long long Clock::current_time_in_microseconds(bool absolute /*= false*/)
  {
#ifdef __MACH__
    mach_timespec_t ts;
    clock_serv_t cclock;
    host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock);
    clock_get_time(cclock, &ts);
    mach_port_deallocate(mach_task_self(), cclock);
#else
    struct timespec ts;
    clock_gettime(absolute ? CLOCK_REALTIME : CLOCK_MONOTONIC, &ts);
#endif
    long long t = (1000000LL * ts.tv_sec) + (ts.tv_nsec / 1000);
    if(!absolute)
      t -= zero_time / 1000;
    return t;
  }

  inline /*static*/ long long Clock::current_time_in_nanoseconds(bool absolute /*= false*/)
  {
#ifdef __MACH__
    mach_timespec_t ts;
    clock_serv_t cclock;
    host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock);
    clock_get_time(cclock, &ts);
    mach_port_deallocate(mach_task_self(), cclock);
#else
    struct timespec ts;
    clock_gettime(absolute ? CLOCK_REALTIME : CLOCK_MONOTONIC, &ts);
#endif
    long long t = (1000000000LL * ts.tv_sec) + ts.tv_nsec;
    if(!absolute)
      t -= zero_time;
    return t;
  }

  inline /*static*/ long long Clock::get_zero_time(void)
  {
    return zero_time;
  }

  inline /*static*/ void Clock::set_zero_time(void)
  {
    // this looks weird, but we can't use the absolute time because it uses
    //  a different system clock in POSIX-land, so we ask for the current
    //  relative time (based on whatever zero_time currently is) and add
    //  that in
    zero_time += current_time_in_nanoseconds(false);
  }

  ////////////////////////////////////////////////////////////////////////
  //
  // class Clock

  inline TimeStamp::TimeStamp(const char *_message, bool _difference, Logger *_logger /*= 0*/)
    : message(_message), difference(_difference), logger(_logger)
  {
    start_time = Clock::current_time();

    if(!difference) {
      if(logger)
	logger->info("%s %7.6f", message, start_time);
      else
	printf("%s %7.6f\n", message, start_time);
    }
  }

  inline TimeStamp::~TimeStamp(void)
  {
    if(difference) {
      double interval = Clock::current_time() - start_time;

      if(logger)
	logger->info("%s %7.6f", message, interval);
      else
	printf("%s %7.6f\n", message, interval);
    }
  }

}; // namespace Realm

<|endoftext|>


<commit_before>/*
 * kmp_stub.cpp -- stub versions of user-callable OpenMP RT functions.
 */

//===----------------------------------------------------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is dual licensed under the MIT and the University of Illinois Open
// Source Licenses. See LICENSE.txt for details.
//
//===----------------------------------------------------------------------===//

#include <errno.h>
#include <limits.h>
#include <stdlib.h>

#include "kmp.h" // KMP_DEFAULT_STKSIZE
#include "kmp_stub.h"
#include "omp.h" // Function renamings.

#if KMP_OS_WINDOWS
#include <windows.h>
#else
#include <sys/time.h>
#endif

// Moved from omp.h
#define omp_set_max_active_levels ompc_set_max_active_levels
#define omp_set_schedule ompc_set_schedule
#define omp_get_ancestor_thread_num ompc_get_ancestor_thread_num
#define omp_get_team_size ompc_get_team_size

#define omp_set_num_threads ompc_set_num_threads
#define omp_set_dynamic ompc_set_dynamic
#define omp_set_nested ompc_set_nested
#define kmp_set_stacksize kmpc_set_stacksize
#define kmp_set_stacksize_s kmpc_set_stacksize_s
#define kmp_set_blocktime kmpc_set_blocktime
#define kmp_set_library kmpc_set_library
#define kmp_set_defaults kmpc_set_defaults
#define kmp_set_disp_num_buffers kmpc_set_disp_num_buffers
#define kmp_malloc kmpc_malloc
#define kmp_aligned_malloc kmpc_aligned_malloc
#define kmp_calloc kmpc_calloc
#define kmp_realloc kmpc_realloc
#define kmp_free kmpc_free

static double frequency = 0.0;

// Helper functions.
static size_t __kmps_init() {
  static int initialized = 0;
  static size_t dummy = 0;
  if (!initialized) {
    // TODO: Analyze KMP_VERSION environment variable, print
    // __kmp_version_copyright and __kmp_version_build_time.
    // WARNING: Do not use "fprintf(stderr, ...)" because it will cause
    // unresolved "__iob" symbol (see C70080). We need to extract __kmp_printf()
    // stuff from kmp_runtime.cpp and use it.

    // Trick with dummy variable forces linker to keep __kmp_version_copyright
    // and __kmp_version_build_time strings in executable file (in case of
    // static linkage). When KMP_VERSION analysis is implemented, dummy
    // variable should be deleted, function should return void.
    dummy = __kmp_version_copyright - __kmp_version_build_time;

#if KMP_OS_WINDOWS
    LARGE_INTEGER freq;
    BOOL status = QueryPerformanceFrequency(&freq);
    if (status) {
      frequency = double(freq.QuadPart);
    }
#endif

    initialized = 1;
  }
  return dummy;
} // __kmps_init

#define i __kmps_init();

/* set API functions */
void omp_set_num_threads(omp_int_t num_threads) { i; }
void omp_set_dynamic(omp_int_t dynamic) {
  i;
  __kmps_set_dynamic(dynamic);
}
void omp_set_nested(omp_int_t nested) {
  i;
  __kmps_set_nested(nested);
}
void omp_set_max_active_levels(omp_int_t max_active_levels) { i; }
void omp_set_schedule(omp_sched_t kind, omp_int_t modifier) {
  i;
  __kmps_set_schedule((kmp_sched_t)kind, modifier);
}
int omp_get_ancestor_thread_num(omp_int_t level) {
  i;
  return (level) ? (-1) : (0);
}
int omp_get_team_size(omp_int_t level) {
  i;
  return (level) ? (-1) : (1);
}
int kmpc_set_affinity_mask_proc(int proc, void **mask) {
  i;
  return -1;
}
int kmpc_unset_affinity_mask_proc(int proc, void **mask) {
  i;
  return -1;
}
int kmpc_get_affinity_mask_proc(int proc, void **mask) {
  i;
  return -1;
}

/* kmp API functions */
void kmp_set_stacksize(omp_int_t arg) {
  i;
  __kmps_set_stacksize(arg);
}
void kmp_set_stacksize_s(size_t arg) {
  i;
  __kmps_set_stacksize(arg);
}
void kmp_set_blocktime(omp_int_t arg) {
  i;
  __kmps_set_blocktime(arg);
}
void kmp_set_library(omp_int_t arg) {
  i;
  __kmps_set_library(arg);
}
void kmp_set_defaults(char const *str) { i; }
void kmp_set_disp_num_buffers(omp_int_t arg) { i; }

/* KMP memory management functions. */
void *kmp_malloc(size_t size) {
  i;
  void *res;
#if KMP_OS_WINDOWS
  // If succesfull returns a pointer to the memory block, otherwise returns
  // NULL.
  // Sets errno to ENOMEM or EINVAL if memory allocation failed or parameter
  // validation failed.
  res = _aligned_malloc(size, 1);
#else
  res = malloc(size);
#endif
  return res;
}
void *kmp_aligned_malloc(size_t sz, size_t a) {
  i;
  int err;
  void *res;
#if KMP_OS_WINDOWS
  res = _aligned_malloc(sz, a);
#else
  if (err = posix_memalign(&res, a, sz)) {
    errno = err; // can be EINVAL or ENOMEM
    res = NULL;
  }
#endif
  return res;
}
void *kmp_calloc(size_t nelem, size_t elsize) {
  i;
  void *res;
#if KMP_OS_WINDOWS
  res = _aligned_recalloc(NULL, nelem, elsize, 1);
#else
  res = calloc(nelem, elsize);
#endif
  return res;
}
void *kmp_realloc(void *ptr, size_t size) {
  i;
  void *res;
#if KMP_OS_WINDOWS
  res = _aligned_realloc(ptr, size, 1);
#else
  res = realloc(ptr, size);
#endif
  return res;
}
void kmp_free(void *ptr) {
  i;
#if KMP_OS_WINDOWS
  _aligned_free(ptr);
#else
  free(ptr);
#endif
}

static int __kmps_blocktime = INT_MAX;

void __kmps_set_blocktime(int arg) {
  i;
  __kmps_blocktime = arg;
} // __kmps_set_blocktime

int __kmps_get_blocktime(void) {
  i;
  return __kmps_blocktime;
} // __kmps_get_blocktime

static int __kmps_dynamic = 0;

void __kmps_set_dynamic(int arg) {
  i;
  __kmps_dynamic = arg;
} // __kmps_set_dynamic

int __kmps_get_dynamic(void) {
  i;
  return __kmps_dynamic;
} // __kmps_get_dynamic

static int __kmps_library = 1000;

void __kmps_set_library(int arg) {
  i;
  __kmps_library = arg;
} // __kmps_set_library

int __kmps_get_library(void) {
  i;
  return __kmps_library;
} // __kmps_get_library

static int __kmps_nested = 0;

void __kmps_set_nested(int arg) {
  i;
  __kmps_nested = arg;
} // __kmps_set_nested

int __kmps_get_nested(void) {
  i;
  return __kmps_nested;
} // __kmps_get_nested

static size_t __kmps_stacksize = KMP_DEFAULT_STKSIZE;

void __kmps_set_stacksize(int arg) {
  i;
  __kmps_stacksize = arg;
} // __kmps_set_stacksize

int __kmps_get_stacksize(void) {
  i;
  return __kmps_stacksize;
} // __kmps_get_stacksize

static kmp_sched_t __kmps_sched_kind = kmp_sched_default;
static int __kmps_sched_modifier = 0;

void __kmps_set_schedule(kmp_sched_t kind, int modifier) {
  i;
  __kmps_sched_kind = kind;
  __kmps_sched_modifier = modifier;
} // __kmps_set_schedule

void __kmps_get_schedule(kmp_sched_t *kind, int *modifier) {
  i;
  *kind = __kmps_sched_kind;
  *modifier = __kmps_sched_modifier;
} // __kmps_get_schedule

#if OMP_40_ENABLED

static kmp_proc_bind_t __kmps_proc_bind = proc_bind_false;

void __kmps_set_proc_bind(kmp_proc_bind_t arg) {
  i;
  __kmps_proc_bind = arg;
} // __kmps_set_proc_bind

kmp_proc_bind_t __kmps_get_proc_bind(void) {
  i;
  return __kmps_proc_bind;
} // __kmps_get_proc_bind

#endif /* OMP_40_ENABLED */

double __kmps_get_wtime(void) {
  // Elapsed wall clock time (in second) from "sometime in the past".
  double wtime = 0.0;
  i;
#if KMP_OS_WINDOWS
  if (frequency > 0.0) {
    LARGE_INTEGER now;
    BOOL status = QueryPerformanceCounter(&now);
    if (status) {
      wtime = double(now.QuadPart) / frequency;
    }
  }
#else
  // gettimeofday() returns seconds and microseconds since the Epoch.
  struct timeval tval;
  int rc;
  rc = gettimeofday(&tval, NULL);
  if (rc == 0) {
    wtime = (double)(tval.tv_sec) + 1.0E-06 * (double)(tval.tv_usec);
  } else {
    // TODO: Assert or abort here.
  }
#endif
  return wtime;
} // __kmps_get_wtime

double __kmps_get_wtick(void) {
  // Number of seconds between successive clock ticks.
  double wtick = 0.0;
  i;
#if KMP_OS_WINDOWS
  {
    DWORD increment;
    DWORD adjustment;
    BOOL disabled;
    BOOL rc;
    rc = GetSystemTimeAdjustment(&adjustment, &increment, &disabled);
    if (rc) {
      wtick = 1.0E-07 * (double)(disabled ? increment : adjustment);
    } else {
      // TODO: Assert or abort here.
      wtick = 1.0E-03;
    }
  }
#else
  // TODO: gettimeofday() returns in microseconds, but what the precision?
  wtick = 1.0E-06;
#endif
  return wtick;
} // __kmps_get_wtick

// end of file //
<commit_msg>Fixed build of the OpenMP stubs library.<commit_after>/*
 * kmp_stub.cpp -- stub versions of user-callable OpenMP RT functions.
 */

//===----------------------------------------------------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is dual licensed under the MIT and the University of Illinois Open
// Source Licenses. See LICENSE.txt for details.
//
//===----------------------------------------------------------------------===//

#include <errno.h>
#include <limits.h>
#include <stdlib.h>

#include "omp.h" // omp_* declarations, must be included before "kmp.h"
#include "kmp.h" // KMP_DEFAULT_STKSIZE
#include "kmp_stub.h"

#if KMP_OS_WINDOWS
#include <windows.h>
#else
#include <sys/time.h>
#endif

// Moved from omp.h
#define omp_set_max_active_levels ompc_set_max_active_levels
#define omp_set_schedule ompc_set_schedule
#define omp_get_ancestor_thread_num ompc_get_ancestor_thread_num
#define omp_get_team_size ompc_get_team_size

#define omp_set_num_threads ompc_set_num_threads
#define omp_set_dynamic ompc_set_dynamic
#define omp_set_nested ompc_set_nested
#define kmp_set_stacksize kmpc_set_stacksize
#define kmp_set_stacksize_s kmpc_set_stacksize_s
#define kmp_set_blocktime kmpc_set_blocktime
#define kmp_set_library kmpc_set_library
#define kmp_set_defaults kmpc_set_defaults
#define kmp_set_disp_num_buffers kmpc_set_disp_num_buffers
#define kmp_malloc kmpc_malloc
#define kmp_aligned_malloc kmpc_aligned_malloc
#define kmp_calloc kmpc_calloc
#define kmp_realloc kmpc_realloc
#define kmp_free kmpc_free

static double frequency = 0.0;

// Helper functions.
static size_t __kmps_init() {
  static int initialized = 0;
  static size_t dummy = 0;
  if (!initialized) {
    // TODO: Analyze KMP_VERSION environment variable, print
    // __kmp_version_copyright and __kmp_version_build_time.
    // WARNING: Do not use "fprintf(stderr, ...)" because it will cause
    // unresolved "__iob" symbol (see C70080). We need to extract __kmp_printf()
    // stuff from kmp_runtime.cpp and use it.

    // Trick with dummy variable forces linker to keep __kmp_version_copyright
    // and __kmp_version_build_time strings in executable file (in case of
    // static linkage). When KMP_VERSION analysis is implemented, dummy
    // variable should be deleted, function should return void.
    dummy = __kmp_version_copyright - __kmp_version_build_time;

#if KMP_OS_WINDOWS
    LARGE_INTEGER freq;
    BOOL status = QueryPerformanceFrequency(&freq);
    if (status) {
      frequency = double(freq.QuadPart);
    }
#endif

    initialized = 1;
  }
  return dummy;
} // __kmps_init

#define i __kmps_init();

/* set API functions */
void omp_set_num_threads(omp_int_t num_threads) { i; }
void omp_set_dynamic(omp_int_t dynamic) {
  i;
  __kmps_set_dynamic(dynamic);
}
void omp_set_nested(omp_int_t nested) {
  i;
  __kmps_set_nested(nested);
}
void omp_set_max_active_levels(omp_int_t max_active_levels) { i; }
void omp_set_schedule(omp_sched_t kind, omp_int_t modifier) {
  i;
  __kmps_set_schedule((kmp_sched_t)kind, modifier);
}
int omp_get_ancestor_thread_num(omp_int_t level) {
  i;
  return (level) ? (-1) : (0);
}
int omp_get_team_size(omp_int_t level) {
  i;
  return (level) ? (-1) : (1);
}
int kmpc_set_affinity_mask_proc(int proc, void **mask) {
  i;
  return -1;
}
int kmpc_unset_affinity_mask_proc(int proc, void **mask) {
  i;
  return -1;
}
int kmpc_get_affinity_mask_proc(int proc, void **mask) {
  i;
  return -1;
}

/* kmp API functions */
void kmp_set_stacksize(omp_int_t arg) {
  i;
  __kmps_set_stacksize(arg);
}
void kmp_set_stacksize_s(size_t arg) {
  i;
  __kmps_set_stacksize(arg);
}
void kmp_set_blocktime(omp_int_t arg) {
  i;
  __kmps_set_blocktime(arg);
}
void kmp_set_library(omp_int_t arg) {
  i;
  __kmps_set_library(arg);
}
void kmp_set_defaults(char const *str) { i; }
void kmp_set_disp_num_buffers(omp_int_t arg) { i; }

/* KMP memory management functions. */
void *kmp_malloc(size_t size) {
  i;
  void *res;
#if KMP_OS_WINDOWS
  // If succesfull returns a pointer to the memory block, otherwise returns
  // NULL.
  // Sets errno to ENOMEM or EINVAL if memory allocation failed or parameter
  // validation failed.
  res = _aligned_malloc(size, 1);
#else
  res = malloc(size);
#endif
  return res;
}
void *kmp_aligned_malloc(size_t sz, size_t a) {
  i;
  int err;
  void *res;
#if KMP_OS_WINDOWS
  res = _aligned_malloc(sz, a);
#else
  if (err = posix_memalign(&res, a, sz)) {
    errno = err; // can be EINVAL or ENOMEM
    res = NULL;
  }
#endif
  return res;
}
void *kmp_calloc(size_t nelem, size_t elsize) {
  i;
  void *res;
#if KMP_OS_WINDOWS
  res = _aligned_recalloc(NULL, nelem, elsize, 1);
#else
  res = calloc(nelem, elsize);
#endif
  return res;
}
void *kmp_realloc(void *ptr, size_t size) {
  i;
  void *res;
#if KMP_OS_WINDOWS
  res = _aligned_realloc(ptr, size, 1);
#else
  res = realloc(ptr, size);
#endif
  return res;
}
void kmp_free(void *ptr) {
  i;
#if KMP_OS_WINDOWS
  _aligned_free(ptr);
#else
  free(ptr);
#endif
}

static int __kmps_blocktime = INT_MAX;

void __kmps_set_blocktime(int arg) {
  i;
  __kmps_blocktime = arg;
} // __kmps_set_blocktime

int __kmps_get_blocktime(void) {
  i;
  return __kmps_blocktime;
} // __kmps_get_blocktime

static int __kmps_dynamic = 0;

void __kmps_set_dynamic(int arg) {
  i;
  __kmps_dynamic = arg;
} // __kmps_set_dynamic

int __kmps_get_dynamic(void) {
  i;
  return __kmps_dynamic;
} // __kmps_get_dynamic

static int __kmps_library = 1000;

void __kmps_set_library(int arg) {
  i;
  __kmps_library = arg;
} // __kmps_set_library

int __kmps_get_library(void) {
  i;
  return __kmps_library;
} // __kmps_get_library

static int __kmps_nested = 0;

void __kmps_set_nested(int arg) {
  i;
  __kmps_nested = arg;
} // __kmps_set_nested

int __kmps_get_nested(void) {
  i;
  return __kmps_nested;
} // __kmps_get_nested

static size_t __kmps_stacksize = KMP_DEFAULT_STKSIZE;

void __kmps_set_stacksize(int arg) {
  i;
  __kmps_stacksize = arg;
} // __kmps_set_stacksize

int __kmps_get_stacksize(void) {
  i;
  return __kmps_stacksize;
} // __kmps_get_stacksize

static kmp_sched_t __kmps_sched_kind = kmp_sched_default;
static int __kmps_sched_modifier = 0;

void __kmps_set_schedule(kmp_sched_t kind, int modifier) {
  i;
  __kmps_sched_kind = kind;
  __kmps_sched_modifier = modifier;
} // __kmps_set_schedule

void __kmps_get_schedule(kmp_sched_t *kind, int *modifier) {
  i;
  *kind = __kmps_sched_kind;
  *modifier = __kmps_sched_modifier;
} // __kmps_get_schedule

#if OMP_40_ENABLED

static kmp_proc_bind_t __kmps_proc_bind = proc_bind_false;

void __kmps_set_proc_bind(kmp_proc_bind_t arg) {
  i;
  __kmps_proc_bind = arg;
} // __kmps_set_proc_bind

kmp_proc_bind_t __kmps_get_proc_bind(void) {
  i;
  return __kmps_proc_bind;
} // __kmps_get_proc_bind

#endif /* OMP_40_ENABLED */

double __kmps_get_wtime(void) {
  // Elapsed wall clock time (in second) from "sometime in the past".
  double wtime = 0.0;
  i;
#if KMP_OS_WINDOWS
  if (frequency > 0.0) {
    LARGE_INTEGER now;
    BOOL status = QueryPerformanceCounter(&now);
    if (status) {
      wtime = double(now.QuadPart) / frequency;
    }
  }
#else
  // gettimeofday() returns seconds and microseconds since the Epoch.
  struct timeval tval;
  int rc;
  rc = gettimeofday(&tval, NULL);
  if (rc == 0) {
    wtime = (double)(tval.tv_sec) + 1.0E-06 * (double)(tval.tv_usec);
  } else {
    // TODO: Assert or abort here.
  }
#endif
  return wtime;
} // __kmps_get_wtime

double __kmps_get_wtick(void) {
  // Number of seconds between successive clock ticks.
  double wtick = 0.0;
  i;
#if KMP_OS_WINDOWS
  {
    DWORD increment;
    DWORD adjustment;
    BOOL disabled;
    BOOL rc;
    rc = GetSystemTimeAdjustment(&adjustment, &increment, &disabled);
    if (rc) {
      wtick = 1.0E-07 * (double)(disabled ? increment : adjustment);
    } else {
      // TODO: Assert or abort here.
      wtick = 1.0E-03;
    }
  }
#else
  // TODO: gettimeofday() returns in microseconds, but what the precision?
  wtick = 1.0E-06;
#endif
  return wtick;
} // __kmps_get_wtick

// end of file //
<|endoftext|>


<commit_before>#include <cmath>
#include <fstream>
#include <sstream>
#include <thread>

#include "io.hh"
#include "Bigrams.hh"
#include "FactorEncoder.hh"

using namespace std;

void
Bigrams::update_trans_stats(const transitions_t &collected_stats,
                            flt_type weight,
                            transitions_t &trans_stats,
                            map<string, flt_type> &unigram_stats)
{
    for (auto srcit = collected_stats.cbegin(); srcit != collected_stats.cend(); ++srcit) {
        for (auto tgtit = srcit->second.cbegin(); tgtit != srcit->second.cend(); ++tgtit) {
            trans_stats[srcit->first][tgtit->first] += weight * tgtit->second;
            unigram_stats[tgtit->first] += weight * tgtit->second;
        }
    }
}

void
Bigrams::collect_trans_stats(const transitions_t &transitions,
                             const map<string, flt_type> &words,
                             map<string, FactorGraph*> &fg_words,
                             transitions_t &trans_stats,
                             map<string, flt_type> &unigram_stats,
                             bool fb)
{
    trans_stats.clear();
    unigram_stats.clear();

    for (auto it = fg_words.begin(); it != fg_words.end(); ++it) {
        transitions_t word_stats;
        if (fb)
            forward_backward(transitions, *it->second, word_stats);
        else
            viterbi(transitions, *it->second, word_stats);
        update_trans_stats(word_stats, words.at(it->first), trans_stats, unigram_stats);
    }
}

void
Bigrams::threaded_backward(const MultiStringFactorGraph *msfg,
                           const std::vector<flt_type> *fw,
                           const map<string, flt_type> *words,
                           transitions_t *trans_stats,
                           int thread_index,
                           int thread_count,
                           flt_type *total_lp)
{
    *total_lp = 0.0;

    auto it = msfg->string_end_nodes.begin();
    for (int i=0; i<thread_index; i++)
        it++;

    while (true) {
        flt_type lp = backward(*msfg, it->first, *fw, *trans_stats, words->at(it->first));
        (*total_lp) += words->at(it->first) * lp;
        for (int i=0; i<thread_count; i++) {
            it++;
            if (it == msfg->string_end_nodes.end()) break;
        }
        if (it == msfg->string_end_nodes.end()) break;
    }
}

flt_type
Bigrams::collect_trans_stats(const transitions_t &transitions,
                             const map<string, flt_type> &words,
                             MultiStringFactorGraph &msfg,
                             transitions_t &trans_stats,
                             map<string, flt_type> &unigram_stats,
                             bool fb,
                             bool threaded)
{
    trans_stats.clear();
    unigram_stats.clear();

    vector<flt_type> fw(msfg.nodes.size(), MIN_FLOAT);
    fw[0] = 0.0;

    forward(transitions, msfg, fw);

    if (!threaded) {
        flt_type total_lp = 0.0;
        for (auto it = msfg.string_end_nodes.begin(); it != msfg.string_end_nodes.end(); ++it) {
            transitions_t word_stats;
            flt_type lp = backward(msfg, it->first, fw, word_stats);
            total_lp += words.at(it->first) * lp;
            update_trans_stats(word_stats, words.at(it->first), trans_stats, unigram_stats);
        }
        return total_lp;
    }

    flt_type total_lp = 0.0;

    vector<transitions_t*> thread_stats(NUM_THREADS);
    vector<flt_type> total_lps(NUM_THREADS);
    thread thrs[NUM_THREADS];

    for (int i=0; i<NUM_THREADS; i++) {
        thread_stats[i] = new transitions_t;
        thrs[i] = thread(threaded_backward, &msfg, &fw, &words, thread_stats[i], i, NUM_THREADS, &(total_lps[i]));
    }

    for (int i=0; i<NUM_THREADS; i++)
        thrs[i].join();

    for (int i=0; i<NUM_THREADS; i++) {
        total_lp += total_lps[i];
        update_trans_stats(*(thread_stats[i]), 1.0, trans_stats, unigram_stats);
    }

    return total_lp;
}

flt_type
Bigrams::collect_trans_stats(const map<string, flt_type> &vocab,
                             const map<string, flt_type> &words,
                             MultiStringFactorGraph &msfg,
                             transitions_t &trans_stats,
                             map<string, flt_type> &unigram_stats,
                             bool fb)
{
    trans_stats.clear();
    unigram_stats.clear();

    vector<flt_type> fw(msfg.nodes.size(), MIN_FLOAT);
    fw[0] = 0.0;

    forward(vocab, msfg, fw);
    flt_type total_lp = 0.0;

    transitions_t word_stats;
    for (auto it = msfg.string_end_nodes.begin(); it != msfg.string_end_nodes.end(); ++it) {
        total_lp += words.at(it->first) * backward(msfg, it->first, fw, word_stats, words.at(it->first));
    }
    update_trans_stats(word_stats, 1.0, trans_stats, unigram_stats);

    return total_lp;
}

void
Bigrams::normalize(transitions_t &trans_stats,
                   flt_type min_cost)
{
    for (auto srcit = trans_stats.begin(); srcit != trans_stats.end(); ++srcit) {

        flt_type normalizer = 0.0;
        for (auto tgtit = srcit->second.begin(); tgtit != srcit->second.end(); ++tgtit)
            normalizer += tgtit->second;
        normalizer = log(normalizer);

        for (auto tgtit = srcit->second.begin(); tgtit != srcit->second.end(); ++tgtit) {

            tgtit->second = log(tgtit->second) - normalizer;
            if (!std::isfinite(tgtit->second)) {
                cerr << "transition " << srcit->first << " " << tgtit->first << " value " << tgtit->second << endl;
                exit(0);
            }
            if (tgtit->second < min_cost) tgtit->second = min_cost;
        }
    }
}

void
Bigrams::write_transitions(const transitions_t &transitions,
                           const string &filename)
{
    io::Stream transfile(filename, "w");

    for (auto srcit = transitions.cbegin(); srcit != transitions.cend(); ++srcit)
        for (auto tgtit = srcit->second.cbegin(); tgtit != srcit->second.cend(); ++tgtit)
            fprintf(transfile.file, "%s %s %f\n", srcit->first.c_str(), tgtit->first.c_str(), tgtit->second);

    transfile.close();
}

int
Bigrams::read_transitions(transitions_t &transitions,
                          const string &filename)
{
    ifstream transfile(filename);
    if (!transfile) return -1;

    string line;
    flt_type count;
    int num_trans = 0;
    while (getline(transfile, line)) {
        stringstream ss(line);
        string src, tgt;
        ss >> src;
        ss >> tgt;
        ss >> count;
        transitions[src][tgt] = count;
        num_trans++;
    }
    transfile.close();

    return num_trans;
}

flt_type
Bigrams::bigram_cost(const transitions_t &transitions,
                     const transitions_t &trans_stats)
{
    flt_type total = 0.0;
    flt_type tmp = 0.0;

    for (auto srcit = trans_stats.cbegin(); srcit != trans_stats.cend(); ++srcit)
        for (auto tgtit = srcit->second.cbegin(); tgtit != srcit->second.cend(); ++tgtit) {
            if (std::isfinite(tgtit->second))
                tmp = tgtit->second * transitions.at(srcit->first).at(tgtit->first);
            if (std::isfinite(tmp)) total += tmp;
            if (!std::isfinite(tmp)) cout << srcit->first << " " << tgtit->first << ": "
                    << tgtit->second << " " << transitions.at(srcit->first).at(tgtit->first) << endl;
        }

    return total;
}

void
Bigrams::remove_transitions(vector<string> &to_remove,
                            transitions_t &transitions)
{
    for (auto it = to_remove.begin(); it != to_remove.end(); ++it)
        transitions.erase(*it);

    for (auto srcit = transitions.begin(); srcit != transitions.end(); ++srcit)
        for (auto it = to_remove.begin(); it != to_remove.end(); ++it)
            srcit->second.erase(*it);

    for (auto srcit = transitions.begin(); srcit != transitions.end();)
        if (srcit->second.size() == 0) transitions.erase(srcit++);
        else ++srcit;
}

int
Bigrams::cutoff(const map<string, flt_type> &unigram_stats,
                flt_type cutoff,
                transitions_t &transitions,
                map<string, FactorGraph*> &fg_words)
{
    vector<string> to_remove;
    for (auto it = unigram_stats.begin(); it != unigram_stats.end(); ++it)
        if (it->second < cutoff && it->first.length() > 2) to_remove.push_back(it->first);

    remove_transitions(to_remove, transitions);

    for (auto fgit = fg_words.begin(); fgit != fg_words.end(); ++fgit) {
        for (int i=0; i<to_remove.size(); i++) {
            size_t found = fgit->first.find(to_remove[i]);
            if (found != std::string::npos) fgit->second->remove_arcs(to_remove[i]);
        }
    }

    return to_remove.size();
}

int
Bigrams::cutoff(const map<string, flt_type> &unigram_stats,
                flt_type cutoff,
                transitions_t &transitions,
                MultiStringFactorGraph &msfg)
{
    vector<string> to_remove;
    for (auto it = unigram_stats.begin(); it != unigram_stats.end(); ++it)
        if (it->second < cutoff && it->first.length() > 2) to_remove.push_back(it->first);

    remove_transitions(to_remove, transitions);

    for (int i=0; i<to_remove.size(); i++)
        msfg.remove_arcs(to_remove[i]);
    msfg.prune_unreachable();

    return to_remove.size();
}

int
Bigrams::remove_least_common(const map<string, flt_type> &unigram_stats,
                             int num_removals,
                             transitions_t &transitions,
                             map<string, FactorGraph*> &fg_words)
{
    vector<pair<string, flt_type> > sorted_stats;
    sort_vocab(unigram_stats, sorted_stats, false);
    vector<string> to_remove;
    for (auto it = sorted_stats.begin(); it != sorted_stats.end(); ++it) {
        if (it->first.length() < 3) continue;
        to_remove.push_back(it->first);
        if (to_remove.size() >= num_removals) break;
    }

    remove_transitions(to_remove, transitions);

    for (auto fgit = fg_words.begin(); fgit != fg_words.end(); ++fgit) {
        for (int i=0; i<num_removals; i++) {
            size_t found = fgit->first.find(sorted_stats[i].first);
            if (found != std::string::npos) fgit->second->remove_arcs(sorted_stats[i].first);
        }
    }

    return to_remove.size();
}

int
Bigrams::remove_least_common(const map<string, flt_type> &unigram_stats,
                             int num_removals,
                             transitions_t &transitions,
                             MultiStringFactorGraph &msfg)
{
    vector<pair<string, flt_type> > sorted_stats;
    sort_vocab(unigram_stats, sorted_stats, false);
    vector<string> to_remove;
    for (auto it = sorted_stats.begin(); it != sorted_stats.end(); ++it) {
        if (it->first.length() < 3) continue;
        to_remove.push_back(it->first);
        if (to_remove.size() >= num_removals) break;
    }

    remove_transitions(to_remove, transitions);

    for (int i=0; i<to_remove.size(); i++)
        msfg.remove_arcs(to_remove[i]);
    msfg.prune_unreachable();

    return to_remove.size();
}

int
Bigrams::transition_count(const transitions_t &transitions)
{
    int count = 0;
    for (auto srcit = transitions.cbegin(); srcit != transitions.cend(); ++srcit)
        for (auto tgtit = srcit->second.cbegin(); tgtit != srcit->second.cend(); ++tgtit)
            count++;
    return count;
}

void
Bigrams::trans_to_vocab(const transitions_t &transitions,
                        map<string, flt_type> &vocab)
{
    vocab.clear();
    for (auto srcit = transitions.cbegin(); srcit != transitions.cend(); ++srcit)
        vocab[srcit->first] = 0.0;
}

<commit_msg>Minor cleanups.<commit_after>#include <cmath>
#include <fstream>
#include <sstream>
#include <thread>

#include "io.hh"
#include "Bigrams.hh"
#include "FactorEncoder.hh"

using namespace std;

void
Bigrams::update_trans_stats(const transitions_t &collected_stats,
                            flt_type weight,
                            transitions_t &trans_stats,
                            map<string, flt_type> &unigram_stats)
{
    for (auto srcit = collected_stats.cbegin(); srcit != collected_stats.cend(); ++srcit) {
        for (auto tgtit = srcit->second.cbegin(); tgtit != srcit->second.cend(); ++tgtit) {
            trans_stats[srcit->first][tgtit->first] += weight * tgtit->second;
            unigram_stats[tgtit->first] += weight * tgtit->second;
        }
    }
}

void
Bigrams::collect_trans_stats(const transitions_t &transitions,
                             const map<string, flt_type> &words,
                             map<string, FactorGraph*> &fg_words,
                             transitions_t &trans_stats,
                             map<string, flt_type> &unigram_stats,
                             bool fb)
{
    trans_stats.clear();
    unigram_stats.clear();

    for (auto it = fg_words.begin(); it != fg_words.end(); ++it) {
        transitions_t word_stats;
        if (fb)
            forward_backward(transitions, *it->second, word_stats);
        else
            viterbi(transitions, *it->second, word_stats);
        update_trans_stats(word_stats, words.at(it->first), trans_stats, unigram_stats);
    }
}

void
Bigrams::threaded_backward(const MultiStringFactorGraph *msfg,
                           const std::vector<flt_type> *fw,
                           const map<string, flt_type> *words,
                           transitions_t *trans_stats,
                           int thread_index,
                           int thread_count,
                           flt_type *total_lp)
{
    *total_lp = 0.0;

    auto it = msfg->string_end_nodes.begin();
    for (int i=0; i<thread_index; i++)
        it++;

    while (true) {
        flt_type lp = backward(*msfg, it->first, *fw, *trans_stats, words->at(it->first));
        (*total_lp) += words->at(it->first) * lp;
        for (int i=0; i<thread_count; i++) {
            it++;
            if (it == msfg->string_end_nodes.end()) break;
        }
        if (it == msfg->string_end_nodes.end()) break;
    }
}

flt_type
Bigrams::collect_trans_stats(const transitions_t &transitions,
                             const map<string, flt_type> &words,
                             MultiStringFactorGraph &msfg,
                             transitions_t &trans_stats,
                             map<string, flt_type> &unigram_stats,
                             bool fb,
                             bool threaded)
{
    trans_stats.clear();
    unigram_stats.clear();

    vector<flt_type> fw(msfg.nodes.size(), MIN_FLOAT);
    fw[0] = 0.0;

    forward(transitions, msfg, fw);

    flt_type total_lp = 0.0;

    if (!threaded) {
        for (auto it = msfg.string_end_nodes.begin(); it != msfg.string_end_nodes.end(); ++it) {
            transitions_t word_stats;
            flt_type lp = backward(msfg, it->first, fw, word_stats);
            total_lp += words.at(it->first) * lp;
            update_trans_stats(word_stats, words.at(it->first), trans_stats, unigram_stats);
        }
        return total_lp;
    }

    vector<transitions_t*> thread_stats(NUM_THREADS);
    vector<flt_type> total_lps(NUM_THREADS);
    thread thrs[NUM_THREADS];

    for (int i=0; i<NUM_THREADS; i++) {
        thread_stats[i] = new transitions_t;
        thrs[i] = thread(threaded_backward, &msfg, &fw, &words, thread_stats[i], i, NUM_THREADS, &(total_lps[i]));
    }

    for (int i=0; i<NUM_THREADS; i++) {
        thrs[i].join();
        total_lp += total_lps[i];
        update_trans_stats(*(thread_stats[i]), 1.0, trans_stats, unigram_stats);
    }

    return total_lp;
}

flt_type
Bigrams::collect_trans_stats(const map<string, flt_type> &vocab,
                             const map<string, flt_type> &words,
                             MultiStringFactorGraph &msfg,
                             transitions_t &trans_stats,
                             map<string, flt_type> &unigram_stats,
                             bool fb)
{
    trans_stats.clear();
    unigram_stats.clear();

    vector<flt_type> fw(msfg.nodes.size(), MIN_FLOAT);
    fw[0] = 0.0;

    forward(vocab, msfg, fw);
    flt_type total_lp = 0.0;

    transitions_t word_stats;
    for (auto it = msfg.string_end_nodes.begin(); it != msfg.string_end_nodes.end(); ++it) {
        total_lp += words.at(it->first) * backward(msfg, it->first, fw, word_stats, words.at(it->first));
    }
    update_trans_stats(word_stats, 1.0, trans_stats, unigram_stats);

    return total_lp;
}

void
Bigrams::normalize(transitions_t &trans_stats,
                   flt_type min_cost)
{
    for (auto srcit = trans_stats.begin(); srcit != trans_stats.end(); ++srcit) {

        flt_type normalizer = 0.0;
        for (auto tgtit = srcit->second.begin(); tgtit != srcit->second.end(); ++tgtit)
            normalizer += tgtit->second;
        normalizer = log(normalizer);

        for (auto tgtit = srcit->second.begin(); tgtit != srcit->second.end(); ++tgtit) {

            tgtit->second = log(tgtit->second) - normalizer;
            if (!std::isfinite(tgtit->second)) {
                cerr << "transition " << srcit->first << " " << tgtit->first << " value " << tgtit->second << endl;
                exit(0);
            }
            if (tgtit->second < min_cost) tgtit->second = min_cost;
        }
    }
}

void
Bigrams::write_transitions(const transitions_t &transitions,
                           const string &filename)
{
    io::Stream transfile(filename, "w");

    for (auto srcit = transitions.cbegin(); srcit != transitions.cend(); ++srcit)
        for (auto tgtit = srcit->second.cbegin(); tgtit != srcit->second.cend(); ++tgtit)
            fprintf(transfile.file, "%s %s %f\n", srcit->first.c_str(), tgtit->first.c_str(), tgtit->second);

    transfile.close();
}

int
Bigrams::read_transitions(transitions_t &transitions,
                          const string &filename)
{
    ifstream transfile(filename);
    if (!transfile) return -1;

    string line;
    flt_type count;
    int num_trans = 0;
    while (getline(transfile, line)) {
        stringstream ss(line);
        string src, tgt;
        ss >> src;
        ss >> tgt;
        ss >> count;
        transitions[src][tgt] = count;
        num_trans++;
    }
    transfile.close();

    return num_trans;
}

flt_type
Bigrams::bigram_cost(const transitions_t &transitions,
                     const transitions_t &trans_stats)
{
    flt_type total = 0.0;
    flt_type tmp = 0.0;

    for (auto srcit = trans_stats.cbegin(); srcit != trans_stats.cend(); ++srcit)
        for (auto tgtit = srcit->second.cbegin(); tgtit != srcit->second.cend(); ++tgtit) {
            if (std::isfinite(tgtit->second))
                tmp = tgtit->second * transitions.at(srcit->first).at(tgtit->first);
            if (std::isfinite(tmp)) total += tmp;
            if (!std::isfinite(tmp)) cout << srcit->first << " " << tgtit->first << ": "
                    << tgtit->second << " " << transitions.at(srcit->first).at(tgtit->first) << endl;
        }

    return total;
}

void
Bigrams::remove_transitions(vector<string> &to_remove,
                            transitions_t &transitions)
{
    for (auto it = to_remove.begin(); it != to_remove.end(); ++it)
        transitions.erase(*it);

    for (auto srcit = transitions.begin(); srcit != transitions.end(); ++srcit)
        for (auto it = to_remove.begin(); it != to_remove.end(); ++it)
            srcit->second.erase(*it);

    for (auto srcit = transitions.begin(); srcit != transitions.end();)
        if (srcit->second.size() == 0) transitions.erase(srcit++);
        else ++srcit;
}

int
Bigrams::cutoff(const map<string, flt_type> &unigram_stats,
                flt_type cutoff,
                transitions_t &transitions,
                map<string, FactorGraph*> &fg_words)
{
    vector<string> to_remove;
    for (auto it = unigram_stats.begin(); it != unigram_stats.end(); ++it)
        if (it->second < cutoff && it->first.length() > 2) to_remove.push_back(it->first);

    remove_transitions(to_remove, transitions);

    for (auto fgit = fg_words.begin(); fgit != fg_words.end(); ++fgit) {
        for (int i=0; i<to_remove.size(); i++) {
            size_t found = fgit->first.find(to_remove[i]);
            if (found != std::string::npos) fgit->second->remove_arcs(to_remove[i]);
        }
    }

    return to_remove.size();
}

int
Bigrams::cutoff(const map<string, flt_type> &unigram_stats,
                flt_type cutoff,
                transitions_t &transitions,
                MultiStringFactorGraph &msfg)
{
    vector<string> to_remove;
    for (auto it = unigram_stats.begin(); it != unigram_stats.end(); ++it)
        if (it->second < cutoff && it->first.length() > 2) to_remove.push_back(it->first);

    remove_transitions(to_remove, transitions);

    for (int i=0; i<to_remove.size(); i++)
        msfg.remove_arcs(to_remove[i]);
    msfg.prune_unreachable();

    return to_remove.size();
}

int
Bigrams::remove_least_common(const map<string, flt_type> &unigram_stats,
                             int num_removals,
                             transitions_t &transitions,
                             map<string, FactorGraph*> &fg_words)
{
    vector<pair<string, flt_type> > sorted_stats;
    sort_vocab(unigram_stats, sorted_stats, false);
    vector<string> to_remove;
    for (auto it = sorted_stats.begin(); it != sorted_stats.end(); ++it) {
        if (it->first.length() < 3) continue;
        to_remove.push_back(it->first);
        if (to_remove.size() >= num_removals) break;
    }

    remove_transitions(to_remove, transitions);

    for (auto fgit = fg_words.begin(); fgit != fg_words.end(); ++fgit) {
        for (int i=0; i<num_removals; i++) {
            size_t found = fgit->first.find(sorted_stats[i].first);
            if (found != std::string::npos) fgit->second->remove_arcs(sorted_stats[i].first);
        }
    }

    return to_remove.size();
}

int
Bigrams::remove_least_common(const map<string, flt_type> &unigram_stats,
                             int num_removals,
                             transitions_t &transitions,
                             MultiStringFactorGraph &msfg)
{
    vector<pair<string, flt_type> > sorted_stats;
    sort_vocab(unigram_stats, sorted_stats, false);
    vector<string> to_remove;
    for (auto it = sorted_stats.begin(); it != sorted_stats.end(); ++it) {
        if (it->first.length() < 3) continue;
        to_remove.push_back(it->first);
        if (to_remove.size() >= num_removals) break;
    }

    remove_transitions(to_remove, transitions);

    for (int i=0; i<to_remove.size(); i++)
        msfg.remove_arcs(to_remove[i]);
    msfg.prune_unreachable();

    return to_remove.size();
}

int
Bigrams::transition_count(const transitions_t &transitions)
{
    int count = 0;
    for (auto srcit = transitions.cbegin(); srcit != transitions.cend(); ++srcit)
        for (auto tgtit = srcit->second.cbegin(); tgtit != srcit->second.cend(); ++tgtit)
            count++;
    return count;
}

void
Bigrams::trans_to_vocab(const transitions_t &transitions,
                        map<string, flt_type> &vocab)
{
    vocab.clear();
    for (auto srcit = transitions.cbegin(); srcit != transitions.cend(); ++srcit)
        vocab[srcit->first] = 0.0;
}

<|endoftext|>


<commit_before>#include "miscs.hxx"

#include <vector>
#include <iostream>
#include <string>

namespace algods {
    namespace miscs {

        string zipstring(const string& s) {
            string ret;
            if(s.empty()) {
                return ret;
            }
            auto idx = [](char ch)->int { return ch - 'a';};
            ret.reserve(s.size());
            constexpr int size = 'z' - 'a' + 1;
            int count[size] = {};
            int i = 0;
            ++count[idx(s[i])];
            for(i = 1; i < s.size(); ++i) {
                ++count[idx(s[i])];
                if(s[i] != s[i-1]) {
                    if(count[idx(s[i-1])] > 1) {
                        char ch = static_cast<char>('0' + count[s[i-1]-'a']);
                        ret.push_back(ch);
                    }
                    ret.push_back(s[i-1]);
                    count[idx(s[i-1])] = 0;
                }
            }
            if(count[idx(s[i-1])] != 0) {
                if(count[idx(s[i-1])] > 1) {
                    char ch = static_cast<char>('0' + count[idx(s[i-1])]);
                    ret.push_back(ch);
                }
                ret.push_back(s[i-1]);
            }
            return ret;
        }
    }
}

namespace algods {
    namespace miscs {
        using namespace std;

        namespace unittest {

            int test_judge0() {
                vector<int> v{4,1,2,5,8,7};
                if(judge(v.begin(), v.end(), 8)) {
                    cout << "testjudge0 passed!" << endl;
                    return 0;
                } else {
                    cout << "testjudge0 failed!" << endl;
                    return 1;
                }
            }

            int test_zipstring0() {
                if(zipstring("abcbc") == "abcbc") {
                    cout << "testzipstring0 passed!" << endl;
                    return 0;
                } else {
                    cout << "testzipstrng0 failed!" << endl;
                    return 1;
                }
            }

            int test_zipstring1() {
                if(zipstring("xxxyyyyyyz") == "3x6yz") {
                    cout << "testzipstring1 passed!" << endl;
                    return 0;
                } else {
                    cout << "testzipstrng1 failed!" << endl;
                    return 1;
                }
            }

            int test_zipstring2() {
                if(zipstring("cccddecc") == "3c2de2c") {
                    cout << "testzipstring2 passed!" << endl;
                    return 0;
                } else {
                    cout << "testzipstrng2 failed!" << endl;
                    return 1;
                }
            }

            int test_zipstring3() {
                if(zipstring("adef") == "adef") {
                    cout << "testzipstring3 passed!" << endl;
                    return 0;
                } else {
                    cout << "testzipstrng3 failed!" << endl;
                    return 1;
                }
            }

            int test_zipstring4() {
                if(zipstring("pppppppp") == "8p") {
                    cout << "testzipstring4 passed!" << endl;
                    return 0;
                } else {
                    cout << "testzipstrng4 failed!" << endl;
                    return 1;
                }
            }
        }
    }
}

int main(int argc, char *argv[]) {
    using namespace algods::miscs::unittest;
    test_judge0();
    test_zipstring0();
    test_zipstring1();
    test_zipstring2();
    test_zipstring3();
    test_zipstring4();
    return 0;
}
<commit_msg>a little improvement<commit_after>#include "miscs.hxx"

#include <vector>
#include <iostream>
#include <string>

namespace algods {
    namespace miscs {

        string zipstring(const string& s) {
            string ret;
            if(s.empty()) {
                return ret;
            }
            auto idx = [](char ch)->int { return ch - 'a';};
            ret.reserve(s.size());
            constexpr int size = 'z' - 'a' + 1;
            int count[size] = {};
            int i = 0;
            ++count[idx(s[i])];
            for(i = 1; i < s.size(); ++i) {
                ++count[idx(s[i])];
                if(s[i] != s[i-1]) {
                    if(count[idx(s[i-1])] > 1) {
                        char ch = static_cast<char>('0' + count[idx(s[i-1])]);
                        ret.push_back(ch);
                    }
                    ret.push_back(s[i-1]);
                    count[idx(s[i-1])] = 0;
                }
            }
            if(count[idx(s[i-1])] != 0) {
                if(count[idx(s[i-1])] > 1) {
                    char ch = static_cast<char>('0' + count[idx(s[i-1])]);
                    ret.push_back(ch);
                }
                ret.push_back(s[i-1]);
            }
            return ret;
        }
    }
}

namespace algods {
    namespace miscs {
        using namespace std;

        namespace unittest {

            int test_judge0() {
                vector<int> v{4,1,2,5,8,7};
                if(judge(v.begin(), v.end(), 8)) {
                    cout << "testjudge0 passed!" << endl;
                    return 0;
                } else {
                    cout << "testjudge0 failed!" << endl;
                    return 1;
                }
            }

            int test_zipstring0() {
                if(zipstring("abcbc") == "abcbc") {
                    cout << "testzipstring0 passed!" << endl;
                    return 0;
                } else {
                    cout << "testzipstrng0 failed!" << endl;
                    return 1;
                }
            }

            int test_zipstring1() {
                if(zipstring("xxxyyyyyyz") == "3x6yz") {
                    cout << "testzipstring1 passed!" << endl;
                    return 0;
                } else {
                    cout << "testzipstrng1 failed!" << endl;
                    return 1;
                }
            }

            int test_zipstring2() {
                if(zipstring("cccddecc") == "3c2de2c") {
                    cout << "testzipstring2 passed!" << endl;
                    return 0;
                } else {
                    cout << "testzipstrng2 failed!" << endl;
                    return 1;
                }
            }

            int test_zipstring3() {
                if(zipstring("adef") == "adef") {
                    cout << "testzipstring3 passed!" << endl;
                    return 0;
                } else {
                    cout << "testzipstrng3 failed!" << endl;
                    return 1;
                }
            }

            int test_zipstring4() {
                if(zipstring("pppppppp") == "8p") {
                    cout << "testzipstring4 passed!" << endl;
                    return 0;
                } else {
                    cout << "testzipstrng4 failed!" << endl;
                    return 1;
                }
            }
        }
    }
}

int main(int argc, char *argv[]) {
    using namespace algods::miscs::unittest;
    test_judge0();
    test_zipstring0();
    test_zipstring1();
    test_zipstring2();
    test_zipstring3();
    test_zipstring4();
    return 0;
}
<|endoftext|>


<commit_before>/*
 *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */
#include "test/scenario/audio_stream.h"

#include "absl/memory/memory.h"
#include "test/call_test.h"

#if WEBRTC_ENABLE_PROTOBUF
RTC_PUSH_IGNORING_WUNDEF()
#ifdef WEBRTC_ANDROID_PLATFORM_BUILD
#include "external/webrtc/webrtc/modules/audio_coding/audio_network_adaptor/config.pb.h"
#else
#include "modules/audio_coding/audio_network_adaptor/config.pb.h"
#endif
RTC_POP_IGNORING_WUNDEF()
#endif

namespace webrtc {
namespace test {
namespace {
enum : int {  // The first valid value is 1.
  kTransportSequenceNumberExtensionId = 1,
  kAbsSendTimeExtensionId
};

absl::optional<std::string> CreateAdaptationString(
    AudioStreamConfig::NetworkAdaptation config) {
#if WEBRTC_ENABLE_PROTOBUF

  audio_network_adaptor::config::ControllerManager cont_conf;
  if (config.frame.max_rate_for_60_ms.IsFinite()) {
    auto controller =
        cont_conf.add_controllers()->mutable_frame_length_controller();
    controller->set_fl_decreasing_packet_loss_fraction(
        config.frame.min_packet_loss_for_decrease);
    controller->set_fl_increasing_packet_loss_fraction(
        config.frame.max_packet_loss_for_increase);

    controller->set_fl_20ms_to_60ms_bandwidth_bps(
        config.frame.min_rate_for_20_ms.bps<int32_t>());
    controller->set_fl_60ms_to_20ms_bandwidth_bps(
        config.frame.max_rate_for_60_ms.bps<int32_t>());

    if (config.frame.max_rate_for_120_ms.IsFinite()) {
      controller->set_fl_60ms_to_120ms_bandwidth_bps(
          config.frame.min_rate_for_60_ms.bps<int32_t>());
      controller->set_fl_120ms_to_60ms_bandwidth_bps(
          config.frame.max_rate_for_120_ms.bps<int32_t>());
    }
  }
  cont_conf.add_controllers()->mutable_bitrate_controller();
  std::string config_string = cont_conf.SerializeAsString();
  return config_string;
#else
  RTC_LOG(LS_ERROR) << "audio_network_adaptation is enabled"
                       " but WEBRTC_ENABLE_PROTOBUF is false.\n"
                       "Ignoring settings.";
  return absl::nullopt;
#endif  // WEBRTC_ENABLE_PROTOBUF
}
}  // namespace

SendAudioStream::SendAudioStream(
    CallClient* sender,
    AudioStreamConfig config,
    rtc::scoped_refptr<AudioEncoderFactory> encoder_factory,
    Transport* send_transport)
    : sender_(sender), config_(config) {
  AudioSendStream::Config send_config(send_transport);
  ssrc_ = sender->GetNextAudioSsrc();
  send_config.rtp.ssrc = ssrc_;
  SdpAudioFormat::Parameters sdp_params;
  if (config.source.channels == 2)
    sdp_params["stereo"] = "1";
  if (config.encoder.initial_frame_length != TimeDelta::Millis(20))
    sdp_params["ptime"] =
        std::to_string(config.encoder.initial_frame_length.ms());
  if (config.encoder.enable_dtx)
    sdp_params["usedtx"] = "1";

  // SdpAudioFormat::num_channels indicates that the encoder is capable of
  // stereo, but the actual channel count used is based on the "stereo"
  // parameter.
  send_config.send_codec_spec = AudioSendStream::Config::SendCodecSpec(
      CallTest::kAudioSendPayloadType, {"opus", 48000, 2, sdp_params});
  RTC_DCHECK_LE(config.source.channels, 2);
  send_config.encoder_factory = encoder_factory;

  if (config.encoder.fixed_rate)
    send_config.send_codec_spec->target_bitrate_bps =
        config.encoder.fixed_rate->bps();
  if (!config.adapt.binary_proto.empty()) {
    send_config.audio_network_adaptor_config = config.adapt.binary_proto;
  } else if (config.network_adaptation) {
    send_config.audio_network_adaptor_config =
        CreateAdaptationString(config.adapt);
  }
  if (config.encoder.allocate_bitrate ||
      config.stream.in_bandwidth_estimation) {
    DataRate min_rate = DataRate::Infinity();
    DataRate max_rate = DataRate::Infinity();
    if (config.encoder.fixed_rate) {
      min_rate = *config.encoder.fixed_rate;
      max_rate = *config.encoder.fixed_rate;
    } else {
      min_rate = *config.encoder.min_rate;
      max_rate = *config.encoder.max_rate;
    }
    send_config.min_bitrate_bps = min_rate.bps();
    send_config.max_bitrate_bps = max_rate.bps();
  }

  if (config.stream.in_bandwidth_estimation) {
    send_config.send_codec_spec->transport_cc_enabled = true;
    send_config.rtp.extensions = {{RtpExtension::kTransportSequenceNumberUri,
                                   kTransportSequenceNumberExtensionId}};
  }
  if (config.stream.abs_send_time) {
    send_config.rtp.extensions.push_back(
        {RtpExtension::kAbsSendTimeUri, kAbsSendTimeExtensionId});
  }

  sender_->SendTask([&] {
    send_stream_ = sender_->call_->CreateAudioSendStream(send_config);
    if (field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")) {
      sender->call_->OnAudioTransportOverheadChanged(
          sender_->transport_->packet_overhead().bytes());
    }
  });
}

SendAudioStream::~SendAudioStream() {
  sender_->SendTask(
      [this] { sender_->call_->DestroyAudioSendStream(send_stream_); });
}

void SendAudioStream::Start() {
  sender_->SendTask([this] {
    send_stream_->Start();
    sender_->call_->SignalChannelNetworkState(MediaType::AUDIO, kNetworkUp);
  });
}

void SendAudioStream::Stop() {
  sender_->SendTask([this] { send_stream_->Stop(); });
}

void SendAudioStream::SetMuted(bool mute) {
  send_stream_->SetMuted(mute);
}

ColumnPrinter SendAudioStream::StatsPrinter() {
  return ColumnPrinter::Lambda(
      "audio_target_rate",
      [this](rtc::SimpleStringBuilder& sb) {
        sender_->SendTask([this, &sb] {
          AudioSendStream::Stats stats = send_stream_->GetStats();
          sb.AppendFormat("%.0lf", stats.target_bitrate_bps / 8.0);
        });
      },
      64);
}

ReceiveAudioStream::ReceiveAudioStream(
    CallClient* receiver,
    AudioStreamConfig config,
    SendAudioStream* send_stream,
    rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
    Transport* feedback_transport)
    : receiver_(receiver), config_(config) {
  AudioReceiveStream::Config recv_config;
  recv_config.rtp.local_ssrc = receiver_->GetNextAudioLocalSsrc();
  recv_config.rtcp_send_transport = feedback_transport;
  recv_config.rtp.remote_ssrc = send_stream->ssrc_;
  receiver->ssrc_media_types_[recv_config.rtp.remote_ssrc] = MediaType::AUDIO;
  if (config.stream.in_bandwidth_estimation) {
    recv_config.rtp.transport_cc = true;
    recv_config.rtp.extensions = {{RtpExtension::kTransportSequenceNumberUri,
                                   kTransportSequenceNumberExtensionId}};
  }
  receiver_->AddExtensions(recv_config.rtp.extensions);
  recv_config.decoder_factory = decoder_factory;
  recv_config.decoder_map = {
      {CallTest::kAudioSendPayloadType, {"opus", 48000, 2}}};
  recv_config.sync_group = config.render.sync_group;
  receiver_->SendTask([&] {
    receive_stream_ = receiver_->call_->CreateAudioReceiveStream(recv_config);
  });
}
ReceiveAudioStream::~ReceiveAudioStream() {
  receiver_->SendTask(
      [&] { receiver_->call_->DestroyAudioReceiveStream(receive_stream_); });
}

void ReceiveAudioStream::Start() {
  receiver_->SendTask([&] {
    receive_stream_->Start();
    receiver_->call_->SignalChannelNetworkState(MediaType::AUDIO, kNetworkUp);
  });
}

void ReceiveAudioStream::Stop() {
  receiver_->SendTask([&] { receive_stream_->Stop(); });
}

AudioReceiveStream::Stats ReceiveAudioStream::GetStats() const {
  AudioReceiveStream::Stats result;
  receiver_->SendTask([&] { result = receive_stream_->GetStats(); });
  return result;
}

AudioStreamPair::~AudioStreamPair() = default;

AudioStreamPair::AudioStreamPair(
    CallClient* sender,
    rtc::scoped_refptr<AudioEncoderFactory> encoder_factory,
    CallClient* receiver,
    rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
    AudioStreamConfig config)
    : config_(config),
      send_stream_(sender, config, encoder_factory, sender->transport_.get()),
      receive_stream_(receiver,
                      config,
                      &send_stream_,
                      decoder_factory,
                      receiver->transport_.get()) {}

}  // namespace test
}  // namespace webrtc
<commit_msg>Thread affinity fix for scenario test SetMuted.<commit_after>/*
 *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */
#include "test/scenario/audio_stream.h"

#include "absl/memory/memory.h"
#include "test/call_test.h"

#if WEBRTC_ENABLE_PROTOBUF
RTC_PUSH_IGNORING_WUNDEF()
#ifdef WEBRTC_ANDROID_PLATFORM_BUILD
#include "external/webrtc/webrtc/modules/audio_coding/audio_network_adaptor/config.pb.h"
#else
#include "modules/audio_coding/audio_network_adaptor/config.pb.h"
#endif
RTC_POP_IGNORING_WUNDEF()
#endif

namespace webrtc {
namespace test {
namespace {
enum : int {  // The first valid value is 1.
  kTransportSequenceNumberExtensionId = 1,
  kAbsSendTimeExtensionId
};

absl::optional<std::string> CreateAdaptationString(
    AudioStreamConfig::NetworkAdaptation config) {
#if WEBRTC_ENABLE_PROTOBUF

  audio_network_adaptor::config::ControllerManager cont_conf;
  if (config.frame.max_rate_for_60_ms.IsFinite()) {
    auto controller =
        cont_conf.add_controllers()->mutable_frame_length_controller();
    controller->set_fl_decreasing_packet_loss_fraction(
        config.frame.min_packet_loss_for_decrease);
    controller->set_fl_increasing_packet_loss_fraction(
        config.frame.max_packet_loss_for_increase);

    controller->set_fl_20ms_to_60ms_bandwidth_bps(
        config.frame.min_rate_for_20_ms.bps<int32_t>());
    controller->set_fl_60ms_to_20ms_bandwidth_bps(
        config.frame.max_rate_for_60_ms.bps<int32_t>());

    if (config.frame.max_rate_for_120_ms.IsFinite()) {
      controller->set_fl_60ms_to_120ms_bandwidth_bps(
          config.frame.min_rate_for_60_ms.bps<int32_t>());
      controller->set_fl_120ms_to_60ms_bandwidth_bps(
          config.frame.max_rate_for_120_ms.bps<int32_t>());
    }
  }
  cont_conf.add_controllers()->mutable_bitrate_controller();
  std::string config_string = cont_conf.SerializeAsString();
  return config_string;
#else
  RTC_LOG(LS_ERROR) << "audio_network_adaptation is enabled"
                       " but WEBRTC_ENABLE_PROTOBUF is false.\n"
                       "Ignoring settings.";
  return absl::nullopt;
#endif  // WEBRTC_ENABLE_PROTOBUF
}
}  // namespace

SendAudioStream::SendAudioStream(
    CallClient* sender,
    AudioStreamConfig config,
    rtc::scoped_refptr<AudioEncoderFactory> encoder_factory,
    Transport* send_transport)
    : sender_(sender), config_(config) {
  AudioSendStream::Config send_config(send_transport);
  ssrc_ = sender->GetNextAudioSsrc();
  send_config.rtp.ssrc = ssrc_;
  SdpAudioFormat::Parameters sdp_params;
  if (config.source.channels == 2)
    sdp_params["stereo"] = "1";
  if (config.encoder.initial_frame_length != TimeDelta::Millis(20))
    sdp_params["ptime"] =
        std::to_string(config.encoder.initial_frame_length.ms());
  if (config.encoder.enable_dtx)
    sdp_params["usedtx"] = "1";

  // SdpAudioFormat::num_channels indicates that the encoder is capable of
  // stereo, but the actual channel count used is based on the "stereo"
  // parameter.
  send_config.send_codec_spec = AudioSendStream::Config::SendCodecSpec(
      CallTest::kAudioSendPayloadType, {"opus", 48000, 2, sdp_params});
  RTC_DCHECK_LE(config.source.channels, 2);
  send_config.encoder_factory = encoder_factory;

  if (config.encoder.fixed_rate)
    send_config.send_codec_spec->target_bitrate_bps =
        config.encoder.fixed_rate->bps();
  if (!config.adapt.binary_proto.empty()) {
    send_config.audio_network_adaptor_config = config.adapt.binary_proto;
  } else if (config.network_adaptation) {
    send_config.audio_network_adaptor_config =
        CreateAdaptationString(config.adapt);
  }
  if (config.encoder.allocate_bitrate ||
      config.stream.in_bandwidth_estimation) {
    DataRate min_rate = DataRate::Infinity();
    DataRate max_rate = DataRate::Infinity();
    if (config.encoder.fixed_rate) {
      min_rate = *config.encoder.fixed_rate;
      max_rate = *config.encoder.fixed_rate;
    } else {
      min_rate = *config.encoder.min_rate;
      max_rate = *config.encoder.max_rate;
    }
    send_config.min_bitrate_bps = min_rate.bps();
    send_config.max_bitrate_bps = max_rate.bps();
  }

  if (config.stream.in_bandwidth_estimation) {
    send_config.send_codec_spec->transport_cc_enabled = true;
    send_config.rtp.extensions = {{RtpExtension::kTransportSequenceNumberUri,
                                   kTransportSequenceNumberExtensionId}};
  }
  if (config.stream.abs_send_time) {
    send_config.rtp.extensions.push_back(
        {RtpExtension::kAbsSendTimeUri, kAbsSendTimeExtensionId});
  }

  sender_->SendTask([&] {
    send_stream_ = sender_->call_->CreateAudioSendStream(send_config);
    if (field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")) {
      sender->call_->OnAudioTransportOverheadChanged(
          sender_->transport_->packet_overhead().bytes());
    }
  });
}

SendAudioStream::~SendAudioStream() {
  sender_->SendTask(
      [this] { sender_->call_->DestroyAudioSendStream(send_stream_); });
}

void SendAudioStream::Start() {
  sender_->SendTask([this] {
    send_stream_->Start();
    sender_->call_->SignalChannelNetworkState(MediaType::AUDIO, kNetworkUp);
  });
}

void SendAudioStream::Stop() {
  sender_->SendTask([this] { send_stream_->Stop(); });
}

void SendAudioStream::SetMuted(bool mute) {
  sender_->SendTask([this, mute] { send_stream_->SetMuted(mute); });
}

ColumnPrinter SendAudioStream::StatsPrinter() {
  return ColumnPrinter::Lambda(
      "audio_target_rate",
      [this](rtc::SimpleStringBuilder& sb) {
        sender_->SendTask([this, &sb] {
          AudioSendStream::Stats stats = send_stream_->GetStats();
          sb.AppendFormat("%.0lf", stats.target_bitrate_bps / 8.0);
        });
      },
      64);
}

ReceiveAudioStream::ReceiveAudioStream(
    CallClient* receiver,
    AudioStreamConfig config,
    SendAudioStream* send_stream,
    rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
    Transport* feedback_transport)
    : receiver_(receiver), config_(config) {
  AudioReceiveStream::Config recv_config;
  recv_config.rtp.local_ssrc = receiver_->GetNextAudioLocalSsrc();
  recv_config.rtcp_send_transport = feedback_transport;
  recv_config.rtp.remote_ssrc = send_stream->ssrc_;
  receiver->ssrc_media_types_[recv_config.rtp.remote_ssrc] = MediaType::AUDIO;
  if (config.stream.in_bandwidth_estimation) {
    recv_config.rtp.transport_cc = true;
    recv_config.rtp.extensions = {{RtpExtension::kTransportSequenceNumberUri,
                                   kTransportSequenceNumberExtensionId}};
  }
  receiver_->AddExtensions(recv_config.rtp.extensions);
  recv_config.decoder_factory = decoder_factory;
  recv_config.decoder_map = {
      {CallTest::kAudioSendPayloadType, {"opus", 48000, 2}}};
  recv_config.sync_group = config.render.sync_group;
  receiver_->SendTask([&] {
    receive_stream_ = receiver_->call_->CreateAudioReceiveStream(recv_config);
  });
}
ReceiveAudioStream::~ReceiveAudioStream() {
  receiver_->SendTask(
      [&] { receiver_->call_->DestroyAudioReceiveStream(receive_stream_); });
}

void ReceiveAudioStream::Start() {
  receiver_->SendTask([&] {
    receive_stream_->Start();
    receiver_->call_->SignalChannelNetworkState(MediaType::AUDIO, kNetworkUp);
  });
}

void ReceiveAudioStream::Stop() {
  receiver_->SendTask([&] { receive_stream_->Stop(); });
}

AudioReceiveStream::Stats ReceiveAudioStream::GetStats() const {
  AudioReceiveStream::Stats result;
  receiver_->SendTask([&] { result = receive_stream_->GetStats(); });
  return result;
}

AudioStreamPair::~AudioStreamPair() = default;

AudioStreamPair::AudioStreamPair(
    CallClient* sender,
    rtc::scoped_refptr<AudioEncoderFactory> encoder_factory,
    CallClient* receiver,
    rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
    AudioStreamConfig config)
    : config_(config),
      send_stream_(sender, config, encoder_factory, sender->transport_.get()),
      receive_stream_(receiver,
                      config,
                      &send_stream_,
                      decoder_factory,
                      receiver->transport_.get()) {}

}  // namespace test
}  // namespace webrtc
<|endoftext|>


<commit_before>
#include "runtime/base/noarybase.h"
#include "runtime/core/var_iterators.h"

namespace xqp
{

/////////////////////////////////////////////////////////////////////////////////
//                                                                             //
//  ForVarIterator                                                             //
//                                                                             //
/////////////////////////////////////////////////////////////////////////////////

ForVarIterator::ForVarIterator(xqpString name, yy::location loc, const void* origin)
  :
  NoaryBaseIterator<ForVarIterator>(loc),
  theVarName(name),
  theOrigin(origin)
{
}

Item_t ForVarIterator::nextImpl(PlanState& planState)
{
  ForVarState* state;
  DEFAULT_STACK_INIT(ForVarState, state, planState);
  STACK_PUSH(state->theValue, state);
  STACK_END();
}

void ForVarIterator::resetImpl(PlanState& planState)
{
  // Do not reset the current value of this var. Only reset the duffs line so
  // that when nextImpl() is called again, the same value will be returned to
  // the caller.
  NoaryBaseIterator<ForVarIterator>::resetImpl(planState);
}

void ForVarIterator::releaseResourcesImpl(PlanState& planState)
{
  NoaryBaseIterator<ForVarIterator>::releaseResourcesImpl(planState);

  ForVarState* state;
  GET_STATE(ForVarState, state, planState);
  state->theValue = NULL;
}

void ForVarIterator::bind(Item_t value, PlanState& planState)
{
  ForVarState* state;
  GET_STATE(ForVarState, state, planState);
  state->theValue = value;
}

/////////////////////////////////////////////////////////////////////////////////
//                                                                             //
//  LetVarIterator                                                             //
//                                                                             //
/////////////////////////////////////////////////////////////////////////////////

LetVarIterator::LetVarIterator(xqpString vn, yy::location loc, const void* origin)
  :
  NoaryBaseIterator<LetVarIterator>(loc),
  theVarName(vn),
  theOrigin(origin)
{
}

Item_t LetVarIterator::nextImpl(PlanState& planState)
{
  LetVarState* state;
  GET_STATE(LetVarState, state, planState);
  return state->theSourceIter->next();
}

void LetVarIterator::resetImpl(PlanState& planState)
{
  NoaryBaseIterator<LetVarIterator>::resetImpl(planState);

  LetVarState* state;
  GET_STATE(LetVarState, state, planState);
  if (state->theSourceIter != NULL)
    state->theSourceIter->reset();
}

void LetVarIterator::releaseResourcesImpl(PlanState& planState)
{
  NoaryBaseIterator<LetVarIterator>::releaseResourcesImpl(planState);

  LetVarState* state;
  GET_STATE(LetVarState, state, planState);
  if (state->theSourceIter != NULL)
  {
    state->theSourceIter->close();
    state->theSourceIter = NULL;
  }
}

void LetVarIterator::bind(Iterator_t it, PlanState& planState)
{
  LetVarState* state;
  GET_STATE(LetVarState, state, planState);
  state->theSourceIter = it;
}

}
<commit_msg>Assertion added<commit_after>#include "util/Assert.h"
#include "runtime/base/noarybase.h"
#include "runtime/core/var_iterators.h"

namespace xqp
{

/////////////////////////////////////////////////////////////////////////////////
//                                                                             //
//  ForVarIterator                                                             //
//                                                                             //
/////////////////////////////////////////////////////////////////////////////////

ForVarIterator::ForVarIterator(xqpString name, yy::location loc, const void* origin)
  :
  NoaryBaseIterator<ForVarIterator>(loc),
  theVarName(name),
  theOrigin(origin)
{
}

Item_t ForVarIterator::nextImpl(PlanState& planState)
{
  ForVarState* state;
  DEFAULT_STACK_INIT(ForVarState, state, planState);
  STACK_PUSH(state->theValue, state);
  STACK_END();
}

void ForVarIterator::resetImpl(PlanState& planState)
{
  // Do not reset the current value of this var. Only reset the duffs line so
  // that when nextImpl() is called again, the same value will be returned to
  // the caller.
  NoaryBaseIterator<ForVarIterator>::resetImpl(planState);
}

void ForVarIterator::releaseResourcesImpl(PlanState& planState)
{
  NoaryBaseIterator<ForVarIterator>::releaseResourcesImpl(planState);

  ForVarState* state;
  GET_STATE(ForVarState, state, planState);
  state->theValue = NULL;
}

void ForVarIterator::bind(Item_t value, PlanState& planState)
{
  ForVarState* state;
  GET_STATE(ForVarState, state, planState);
  state->theValue = value;
}

/////////////////////////////////////////////////////////////////////////////////
//                                                                             //
//  LetVarIterator                                                             //
//                                                                             //
/////////////////////////////////////////////////////////////////////////////////

LetVarIterator::LetVarIterator(xqpString vn, yy::location loc, const void* origin)
  :
  NoaryBaseIterator<LetVarIterator>(loc),
  theVarName(vn),
  theOrigin(origin)
{
}

Item_t LetVarIterator::nextImpl(PlanState& planState)
{
  LetVarState* state;
  GET_STATE(LetVarState, state, planState);
  ZORBA_ASSERT (state->theSourceIter != NULL);
  return state->theSourceIter->next();
}

void LetVarIterator::resetImpl(PlanState& planState)
{
  NoaryBaseIterator<LetVarIterator>::resetImpl(planState);

  LetVarState* state;
  GET_STATE(LetVarState, state, planState);
  if (state->theSourceIter != NULL)
    state->theSourceIter->reset();
}

void LetVarIterator::releaseResourcesImpl(PlanState& planState)
{
  NoaryBaseIterator<LetVarIterator>::releaseResourcesImpl(planState);

  LetVarState* state;
  GET_STATE(LetVarState, state, planState);
  if (state->theSourceIter != NULL)
  {
    state->theSourceIter->close();
    state->theSourceIter = NULL;
  }
}

void LetVarIterator::bind(Iterator_t it, PlanState& planState)
{
  LetVarState* state;
  GET_STATE(LetVarState, state, planState);
  state->theSourceIter = it;
}

}
<|endoftext|>


<commit_before>#include "fetest.hh"

#include <cppunit/TestFixture.h>
#include <cppunit/extensions/HelperMacros.h>

CPPUNIT_TEST_SUITE_REGISTRATION (fetest);

void fetest :: setUp (void)
{
}

void fetest :: tearDown (void)
{
}

void fetest :: viterbiTest1 (void)
{
    std::map<std::string, double> vocab;
    std::string str1("a"); std::string str2("bc");
    vocab[str1] = -1.0;
    vocab[str2] = -2.0;
    std::string sentence("abc");
    std::vector<std::string> best_path;
    int maxlen = 2;
    viterbi(vocab, maxlen, sentence, best_path);
    CPPUNIT_ASSERT_EQUAL(2, (int)best_path.size());
    CPPUNIT_ASSERT_EQUAL(str1, best_path[0]);
    CPPUNIT_ASSERT_EQUAL(str2, best_path[1]);
}

void fetest :: viterbiTest2 (void)
{
    std::map<std::string, double> vocab;
    std::string str1("a"); std::string str2("bc");
    std::string str3("ab"); std::string str4("c");
    vocab[str1] = -1.0;
    vocab[str2] = -2.0;
    vocab[str3] = 0.0;
    vocab[str4] = 0.0;
    std::string sentence("abc");
    std::vector<std::string> best_path;
    int maxlen = 2;
    viterbi(vocab, maxlen, sentence, best_path);
    CPPUNIT_ASSERT_EQUAL(2, (int)best_path.size());
    CPPUNIT_ASSERT_EQUAL(str3, best_path[0]);
    CPPUNIT_ASSERT_EQUAL(str4, best_path[1]);
}

// No possible segmentation
void fetest :: viterbiTest3 (void)
{
    std::map<std::string, double> vocab;
    std::string str1("a");
    vocab[str1] = -1.0;
    std::string sentence("abc");
    std::vector<std::string> best_path;
    int maxlen = 1;
    viterbi(vocab, maxlen, sentence, best_path);
    CPPUNIT_ASSERT_EQUAL(0, (int)best_path.size());
}

// Empty string
void fetest :: viterbiTest4 (void)
{
    std::map<std::string, double> vocab;
    std::string str1("a");
    vocab[str1] = -1.0;
    std::string sentence("");
    std::vector<std::string> best_path;
    int maxlen = 1;
    viterbi(vocab, maxlen, sentence, best_path);
    CPPUNIT_ASSERT_EQUAL(0, (int)best_path.size());
}

// One character sentence
void fetest :: viterbiTest5 (void)
{
    std::map<std::string, double> vocab;
    std::string str1("a");
    vocab[str1] = -1.0;
    std::string sentence("a");
    std::vector<std::string> best_path;
    int maxlen = 1;
    viterbi(vocab, maxlen, sentence, best_path);
    CPPUNIT_ASSERT_EQUAL(1, (int)best_path.size());
    CPPUNIT_ASSERT_EQUAL(str1, best_path[0]);
}

// No segmentation
void fetest :: viterbiTest6 (void)
{
    std::map<std::string, double> vocab;
    std::string str1("a"); std::string str2("b");
    std::string str3("c"); std::string str4("d");
    vocab[str1] = -1.0;
    vocab[str2] = -2.0;
    vocab[str3] = 0.0;
    vocab[str4] = 0.0;
    std::string sentence("a-bcd");
    std::vector<std::string> best_path;
    int maxlen = 1;
    viterbi(vocab, maxlen, sentence, best_path);
    CPPUNIT_ASSERT_EQUAL(0, (int)best_path.size());
}

<commit_msg>using std namespace.<commit_after>#include <map>
#include <string>
#include <vector>

#include "fetest.hh"

#include <cppunit/TestFixture.h>
#include <cppunit/extensions/HelperMacros.h>

CPPUNIT_TEST_SUITE_REGISTRATION (fetest);

void fetest :: setUp (void)
{
}

void fetest :: tearDown (void)
{
}

void fetest :: viterbiTest1 (void)
{
    map<string, double> vocab;
    string str1("a"); string str2("bc");
    vocab[str1] = -1.0;
    vocab[str2] = -2.0;
    string sentence("abc");
    vector<string> best_path;
    int maxlen = 2;
    viterbi(vocab, maxlen, sentence, best_path);
    CPPUNIT_ASSERT_EQUAL(2, (int)best_path.size());
    CPPUNIT_ASSERT_EQUAL(str1, best_path[0]);
    CPPUNIT_ASSERT_EQUAL(str2, best_path[1]);
}

void fetest :: viterbiTest2 (void)
{
    map<string, double> vocab;
    string str1("a"); string str2("bc");
    string str3("ab"); string str4("c");
    vocab[str1] = -1.0;
    vocab[str2] = -2.0;
    vocab[str3] = 0.0;
    vocab[str4] = 0.0;
    string sentence("abc");
    vector<string> best_path;
    int maxlen = 2;
    viterbi(vocab, maxlen, sentence, best_path);
    CPPUNIT_ASSERT_EQUAL(2, (int)best_path.size());
    CPPUNIT_ASSERT_EQUAL(str3, best_path[0]);
    CPPUNIT_ASSERT_EQUAL(str4, best_path[1]);
}

// No possible segmentation
void fetest :: viterbiTest3 (void)
{
    map<string, double> vocab;
    string str1("a");
    vocab[str1] = -1.0;
    string sentence("abc");
    vector<string> best_path;
    int maxlen = 1;
    viterbi(vocab, maxlen, sentence, best_path);
    CPPUNIT_ASSERT_EQUAL(0, (int)best_path.size());
}

// Empty string
void fetest :: viterbiTest4 (void)
{
    map<string, double> vocab;
    string str1("a");
    vocab[str1] = -1.0;
    string sentence("");
    vector<string> best_path;
    int maxlen = 1;
    viterbi(vocab, maxlen, sentence, best_path);
    CPPUNIT_ASSERT_EQUAL(0, (int)best_path.size());
}

// One character sentence
void fetest :: viterbiTest5 (void)
{
    map<string, double> vocab;
    string str1("a");
    vocab[str1] = -1.0;
    string sentence("a");
    vector<string> best_path;
    int maxlen = 1;
    viterbi(vocab, maxlen, sentence, best_path);
    CPPUNIT_ASSERT_EQUAL(1, (int)best_path.size());
    CPPUNIT_ASSERT_EQUAL(str1, best_path[0]);
}

// No segmentation
void fetest :: viterbiTest6 (void)
{
    map<string, double> vocab;
    string str1("a"); string str2("b");
    string str3("c"); string str4("d");
    vocab[str1] = -1.0;
    vocab[str2] = -2.0;
    vocab[str3] = 0.0;
    vocab[str4] = 0.0;
    string sentence("a-bcd");
    vector<string> best_path;
    int maxlen = 1;
    viterbi(vocab, maxlen, sentence, best_path);
    CPPUNIT_ASSERT_EQUAL(0, (int)best_path.size());
}

<|endoftext|>