<commit_before>/**************************************************************************
**
** This file is part of Qt Creator
**
** Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
**
** Contact: Nokia Corporation (qt-info@nokia.com)
**
** Commercial Usage
**
** Licensees holding valid Qt Commercial licenses may use this file in
** accordance with the Qt Commercial License Agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Nokia.
**
** GNU Lesser General Public License Usage
**
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** If you are unsure which license is appropriate for your use, please
** contact the sales department at http://qt.nokia.com/contact.
**
**************************************************************************/

#include "detailswidget.h"
#include "detailsbutton.h"

#include <QtCore/QStack>

#include <QtGui/QGridLayout>
#include <QtGui/QLabel>
#include <QtGui/QPainter>

namespace Utils {

static const int MARGIN=8;

// This widget is using a grid layout and places the items
// in the following way:
//
// +------------+-------------------------+---------------+
// + toolWidget | summaryLabel            | detailsButton |
// +------------+-------------------------+---------------+
// |            | widget                                  |
// +------------+-------------------------+---------------+

struct DetailsWidgetPrivate {
    DetailsWidgetPrivate();

    DetailsButton *m_detailsButton;
    QGridLayout *m_grid;
    QLabel *m_summaryLabel;
    QWidget *m_toolWidget;
    QWidget *m_widget;

    QPixmap m_collapsedPixmap;
    QPixmap m_expandedPixmap;

    DetailsWidget::State m_state;
    bool m_hovered;
};

DetailsWidgetPrivate::DetailsWidgetPrivate() :
        m_detailsButton(new DetailsButton),
        m_grid(new QGridLayout),
        m_summaryLabel(new QLabel),
        m_toolWidget(0),
        m_widget(0),
        m_state(DetailsWidget::Collapsed),
        m_hovered(false)
{
}

DetailsWidget::DetailsWidget(QWidget *parent) :
    QWidget(parent),
    d(new DetailsWidgetPrivate)
{
    d->m_summaryLabel->setTextInteractionFlags(Qt::TextSelectableByMouse);
    d->m_summaryLabel->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Fixed);
    d->m_summaryLabel->setContentsMargins(MARGIN, MARGIN, MARGIN, MARGIN);

    d->m_grid->setContentsMargins(0, 0, 0, 0);
    d->m_grid->setSpacing(0);
    d->m_grid->addWidget(d->m_summaryLabel, 0, 1);
    d->m_grid->addWidget(d->m_detailsButton, 0, 2);
    setLayout(d->m_grid);

    connect(d->m_detailsButton, SIGNAL(toggled(bool)),
            this, SLOT(setExpanded(bool)));
    updateControls();
}

DetailsWidget::~DetailsWidget()
{
    delete d;
}

void DetailsWidget::paintEvent(QPaintEvent *paintEvent)
{
    QWidget::paintEvent(paintEvent);

    QPainter p(this);

    QPoint topLeft(d->m_summaryLabel->geometry().left(), contentsRect().top());
    const QRect paintArea(topLeft, contentsRect().bottomRight());

    if (d->m_state != Expanded) {
        if (d->m_collapsedPixmap.isNull() ||
            d->m_collapsedPixmap.size() != size())
            d->m_collapsedPixmap = cacheBackground(paintArea.size(), false);
        p.drawPixmap(paintArea, d->m_collapsedPixmap);
    } else {
        if (d->m_expandedPixmap.isNull() ||
            d->m_expandedPixmap.size() != size())
            d->m_expandedPixmap = cacheBackground(paintArea.size(), true);
        p.drawPixmap(paintArea, d->m_expandedPixmap);
    }
}

void DetailsWidget::enterEvent(QEvent * event)
{
    QWidget::enterEvent(event);
    changeHoverState(true);
}

void DetailsWidget::leaveEvent(QEvent * event)
{
    QWidget::leaveEvent(event);
    changeHoverState(false);
}

void DetailsWidget::setSummaryText(const QString &text)
{
    d->m_summaryLabel->setText(text);
}

QString DetailsWidget::summaryText() const
{
    return d->m_summaryLabel->text();
}

DetailsWidget::State DetailsWidget::state() const
{
    return d->m_state;
}

void DetailsWidget::setState(State state)
{
    if (state == d->m_state)
        return;
    d->m_state = state;
    updateControls();
}

void DetailsWidget::setExpanded(bool expanded)
{
    setState(expanded ? Expanded : Collapsed);
}

void DetailsWidget::updateControls()
{
    if (d->m_widget)
        d->m_widget->setVisible(d->m_state == Expanded || d->m_state == NoSummary);
    d->m_detailsButton->setChecked(d->m_state == Expanded && d->m_widget);
    d->m_summaryLabel->setEnabled(d->m_state == Collapsed && d->m_widget);
    d->m_detailsButton->setVisible(d->m_state != NoSummary);
    d->m_summaryLabel->setVisible(d->m_state != NoSummary);
}

QWidget *DetailsWidget::widget() const
{
    return d->m_widget;
}

void DetailsWidget::setWidget(QWidget *widget)
{
    if (d->m_widget == widget)
        return;

    if (d->m_widget) {
        d->m_grid->removeWidget(d->m_widget);
        delete d->m_widget;
    }

    d->m_widget = widget;

    if (d->m_widget) {
        d->m_widget->setContentsMargins(MARGIN, MARGIN, MARGIN, MARGIN);
        d->m_grid->addWidget(d->m_widget, 1, 1, 1, 2);
    }
    updateControls();
}

void DetailsWidget::setToolWidget(QWidget *widget)
{
    if (d->m_toolWidget == widget)
        return;

    d->m_toolWidget = widget;

    if (!d->m_toolWidget)
        return;

    d->m_toolWidget->adjustSize();
    d->m_grid->addWidget(d->m_toolWidget, 0, 0, 1, 1, Qt::AlignCenter);

    d->m_grid->setColumnMinimumWidth(0, d->m_toolWidget->width());
    d->m_grid->setRowMinimumHeight(0, d->m_toolWidget->height());

    changeHoverState(d->m_hovered);
}

QWidget *DetailsWidget::toolWidget() const
{
    return d->m_toolWidget;
}

QPixmap DetailsWidget::cacheBackground(const QSize &size, bool expanded)
{
    QLinearGradient lg;
    lg.setCoordinateMode(QGradient::ObjectBoundingMode);
    lg.setFinalStop(0, 1);

    lg.setColorAt(0, palette().color(QPalette::Midlight));
    lg.setColorAt(1, palette().color(QPalette::Button));

    QPixmap pixmap(size);
    QPainter p(&pixmap);
    p.setBrush(lg);
    p.setPen(QPen(palette().color(QPalette::Mid)));

    p.drawRect(0, 0, size.width() - 1, size.height() - 1);

    if (expanded) {
        p.drawLine(0, d->m_widget->geometry().top() - 1,
                   d->m_summaryLabel->width(), d->m_widget->geometry().top() - 1);
    }

    return pixmap;
}

void DetailsWidget::changeHoverState(bool hovered)
{
    if (!d->m_toolWidget)
        return;

    d->m_toolWidget->setVisible(hovered);

    d->m_hovered = hovered;
}

} // namespace Utils
<commit_msg>Remove more/less-flicker in the projects tab<commit_after>/**************************************************************************
**
** This file is part of Qt Creator
**
** Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
**
** Contact: Nokia Corporation (qt-info@nokia.com)
**
** Commercial Usage
**
** Licensees holding valid Qt Commercial licenses may use this file in
** accordance with the Qt Commercial License Agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Nokia.
**
** GNU Lesser General Public License Usage
**
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** If you are unsure which license is appropriate for your use, please
** contact the sales department at http://qt.nokia.com/contact.
**
**************************************************************************/

#include "detailswidget.h"
#include "detailsbutton.h"

#include <QtCore/QStack>

#include <QtGui/QGridLayout>
#include <QtGui/QLabel>
#include <QtGui/QPainter>
#include <QtGui/QScrollArea>
#include <QtGui/QApplication>

namespace Utils {

static const int MARGIN=8;

// This widget is using a grid layout and places the items
// in the following way:
//
// +------------+-------------------------+---------------+
// + toolWidget | summaryLabel            | detailsButton |
// +------------+-------------------------+---------------+
// |            | widget                                  |
// +------------+-------------------------+---------------+

struct DetailsWidgetPrivate {
    DetailsWidgetPrivate();

    DetailsButton *m_detailsButton;
    QGridLayout *m_grid;
    QLabel *m_summaryLabel;
    QWidget *m_toolWidget;
    QWidget *m_widget;

    QPixmap m_collapsedPixmap;
    QPixmap m_expandedPixmap;

    DetailsWidget::State m_state;
    bool m_hovered;
};

DetailsWidgetPrivate::DetailsWidgetPrivate() :
        m_detailsButton(new DetailsButton),
        m_grid(new QGridLayout),
        m_summaryLabel(new QLabel),
        m_toolWidget(0),
        m_widget(0),
        m_state(DetailsWidget::Collapsed),
        m_hovered(false)
{
}

DetailsWidget::DetailsWidget(QWidget *parent) :
    QWidget(parent),
    d(new DetailsWidgetPrivate)
{
    d->m_summaryLabel->setTextInteractionFlags(Qt::TextSelectableByMouse);
    d->m_summaryLabel->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Fixed);
    d->m_summaryLabel->setContentsMargins(MARGIN, MARGIN, MARGIN, MARGIN);

    d->m_grid->setContentsMargins(0, 0, 0, 0);
    d->m_grid->setSpacing(0);
    d->m_grid->addWidget(d->m_summaryLabel, 0, 1);
    d->m_grid->addWidget(d->m_detailsButton, 0, 2);
    setLayout(d->m_grid);

    connect(d->m_detailsButton, SIGNAL(toggled(bool)),
            this, SLOT(setExpanded(bool)));
    updateControls();
}

DetailsWidget::~DetailsWidget()
{
    delete d;
}

void DetailsWidget::paintEvent(QPaintEvent *paintEvent)
{
    QWidget::paintEvent(paintEvent);

    QPainter p(this);

    QPoint topLeft(d->m_summaryLabel->geometry().left(), contentsRect().top());
    const QRect paintArea(topLeft, contentsRect().bottomRight());

    if (d->m_state != Expanded) {
        if (d->m_collapsedPixmap.isNull() ||
            d->m_collapsedPixmap.size() != size())
            d->m_collapsedPixmap = cacheBackground(paintArea.size(), false);
        p.drawPixmap(paintArea, d->m_collapsedPixmap);
    } else {
        if (d->m_expandedPixmap.isNull() ||
            d->m_expandedPixmap.size() != size())
            d->m_expandedPixmap = cacheBackground(paintArea.size(), true);
        p.drawPixmap(paintArea, d->m_expandedPixmap);
    }
}

void DetailsWidget::enterEvent(QEvent * event)
{
    QWidget::enterEvent(event);
    changeHoverState(true);
}

void DetailsWidget::leaveEvent(QEvent * event)
{
    QWidget::leaveEvent(event);
    changeHoverState(false);
}

void DetailsWidget::setSummaryText(const QString &text)
{
    d->m_summaryLabel->setText(text);
}

QString DetailsWidget::summaryText() const
{
    return d->m_summaryLabel->text();
}

DetailsWidget::State DetailsWidget::state() const
{
    return d->m_state;
}

void DetailsWidget::setState(State state)
{
    if (state == d->m_state)
        return;
    d->m_state = state;
    updateControls();
}

void DetailsWidget::setExpanded(bool expanded)
{
    setState(expanded ? Expanded : Collapsed);
}

void DetailsWidget::updateControls()
{
    if (d->m_widget)
        d->m_widget->setVisible(d->m_state == Expanded || d->m_state == NoSummary);
    d->m_detailsButton->setChecked(d->m_state == Expanded && d->m_widget);
    d->m_summaryLabel->setEnabled(d->m_state == Collapsed && d->m_widget);
    d->m_detailsButton->setVisible(d->m_state != NoSummary);
    d->m_summaryLabel->setVisible(d->m_state != NoSummary);
    {
        QWidget *w = this;
        while (w) {
            if (w->layout())
                w->layout()->activate();
            if (QScrollArea *area = qobject_cast<QScrollArea*>(w)) {
                QEvent e(QEvent::LayoutRequest);
                QCoreApplication::sendEvent(area, &e);
            }
            w = w->parentWidget();
        }
    }
}

QWidget *DetailsWidget::widget() const
{
    return d->m_widget;
}

void DetailsWidget::setWidget(QWidget *widget)
{
    if (d->m_widget == widget)
        return;

    if (d->m_widget) {
        d->m_grid->removeWidget(d->m_widget);
        delete d->m_widget;
    }

    d->m_widget = widget;

    if (d->m_widget) {
        d->m_widget->setContentsMargins(MARGIN, MARGIN, MARGIN, MARGIN);
        d->m_grid->addWidget(d->m_widget, 1, 1, 1, 2);
    }
    updateControls();
}

void DetailsWidget::setToolWidget(QWidget *widget)
{
    if (d->m_toolWidget == widget)
        return;

    d->m_toolWidget = widget;

    if (!d->m_toolWidget)
        return;

    d->m_toolWidget->adjustSize();
    d->m_grid->addWidget(d->m_toolWidget, 0, 0, 1, 1, Qt::AlignCenter);

    d->m_grid->setColumnMinimumWidth(0, d->m_toolWidget->width());
    d->m_grid->setRowMinimumHeight(0, d->m_toolWidget->height());

    changeHoverState(d->m_hovered);
}

QWidget *DetailsWidget::toolWidget() const
{
    return d->m_toolWidget;
}

QPixmap DetailsWidget::cacheBackground(const QSize &size, bool expanded)
{
    QLinearGradient lg;
    lg.setCoordinateMode(QGradient::ObjectBoundingMode);
    lg.setFinalStop(0, 1);

    lg.setColorAt(0, palette().color(QPalette::Midlight));
    lg.setColorAt(1, palette().color(QPalette::Button));

    QPixmap pixmap(size);
    QPainter p(&pixmap);
    p.setBrush(lg);
    p.setPen(QPen(palette().color(QPalette::Mid)));

    p.drawRect(0, 0, size.width() - 1, size.height() - 1);

    if (expanded) {
        p.drawLine(0, d->m_widget->geometry().top() - 1,
                   d->m_summaryLabel->width(), d->m_widget->geometry().top() - 1);
    }

    return pixmap;
}

void DetailsWidget::changeHoverState(bool hovered)
{
    if (!d->m_toolWidget)
        return;

    d->m_toolWidget->setVisible(hovered);

    d->m_hovered = hovered;
}

} // namespace Utils
<|endoftext|>


<commit_before>// Copyright 2012 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Parses a module and ETW trace files, generating an ordering of the
// blocks in the decomposed image.

#include "syzygy/reorder/reorder_app.h"

#include <objbase.h>
#include <iostream>

#include "base/string_number_conversions.h"
#include "base/stringprintf.h"
#include "base/strings/string_split.h"
#include "syzygy/grinder/basic_block_entry_count_serializer.h"
#include "syzygy/grinder/basic_block_util.h"
#include "syzygy/pe/find.h"
#include "syzygy/reorder/basic_block_optimizer.h"
#include "syzygy/reorder/dead_code_finder.h"
#include "syzygy/reorder/linear_order_generator.h"
#include "syzygy/reorder/random_order_generator.h"

namespace reorder {

namespace {

using grinder::basic_block_util::EntryCountMap;
using grinder::basic_block_util::ModuleEntryCountMap;
using grinder::basic_block_util::LoadBasicBlockRanges;
using grinder::basic_block_util::FindEntryCountMap;
using grinder::basic_block_util::RelativeAddressRangeVector;
using grinder::BasicBlockEntryCountSerializer;

static const char kUsageFormatStr[] =
    "Usage: %ls [options] [log files ...]\n"
    "  Required Options:\n"
    "    --instrumented-image=<path> the path to the instrumented image file.\n"
    "    --output-file=<path> the output file.\n"
    "  Optional Options:\n"
    "    --input-image=<path> the input image file to reorder. If this is not\n"
    "        specified it will be inferred from the instrumented image's\n"
    "        metadata.\n"
    "    --basic-block-entry-counts=PATH the path to the JSON file containing\n"
    "        the summary basic-block entry counts for the image. If this is\n"
    "        given then the input image is also required.\n"
    "    --seed=INT generates a random ordering; don't specify ETW log files.\n"
    "    --list-dead-code instead of an ordering, output the set of functions\n"
    "        not visited during the trace.\n"
    "    --pretty-print enables pretty printing of the JSON output file.\n"
    "    --reorderer-flags=<comma separated reorderer flags>\n"
    "  Reorderer Flags:\n"
    "    no-code: Do not reorder code sections.\n"
    "    no-data: Do not reorder data sections.\n"
    "  Deprecated Options:\n"
    "    --instrumented-dll=<path> aliases to --instrumented-image.\n"
    "    --input-dll=<path> aliases to --input-image.\n";

// Parses reorderer flags. Returns true on success, false otherwise. On
// failure, also outputs Usage with an error message.
bool ParseFlags(const std::string& flags_str, Reorderer::Flags* flags) {
  DCHECK(flags != NULL);

  // Set the default flags value.
  Reorderer::Flags out_flags =
      Reorderer::kFlagReorderData | Reorderer::kFlagReorderCode;

  // If there is a string to process then extract its flags.
  if (!flags_str.empty()) {
    typedef std::vector<std::string> StringVector;
    StringVector text_flags;
    base::SplitString(flags_str, ',', &text_flags);
    StringVector::const_iterator flag_iter = text_flags.begin();
    for (; flag_iter != text_flags.end(); ++flag_iter) {
      if (*flag_iter == "no-data") {
        out_flags &= ~Reorderer::kFlagReorderData;
      } else if (*flag_iter == "no-code") {
        out_flags &= ~Reorderer::kFlagReorderCode;
      } else if (!flag_iter->empty()) {
        LOG(ERROR) << "Unknown reorderer flag: " << *flag_iter << ".";
        return false;
      }
    }
  }

  // Set the return value.
  *flags = out_flags;

  return true;
}

}  // namespace

const char ReorderApp::kInstrumentedImage[] = "instrumented-image";
const char ReorderApp::kOutputFile[] = "output-file";
const char ReorderApp::kInputImage[] = "input-image";
const char ReorderApp::kBasicBlockEntryCounts[] = "basic-block-entry-counts";
const char ReorderApp::kSeed[] = "seed";
const char ReorderApp::kListDeadCode[] = "list-dead-code";
const char ReorderApp::kPrettyPrint[] = "pretty-print";
const char ReorderApp::kReordererFlags[] = "reorderer-flags";
const char ReorderApp::kInstrumentedDll[] = "instrumented-dll";
const char ReorderApp::kInputDll[] = "input-dll";

ReorderApp::ReorderApp()
    : AppImplBase("Reorder"),
      mode_(kInvalidMode),
      seed_(0),
      pretty_print_(false),
      flags_(0) {
}

bool ReorderApp::ParseCommandLine(const CommandLine* command_line) {
  DCHECK(command_line != NULL);
  DCHECK_EQ(kInvalidMode, mode_);

  // Parse the instrumented image path.
  if (!GetDeprecatedSwitch(command_line,
                           kInstrumentedImage,
                           kInstrumentedDll,
                           &CommandLine::GetSwitchValuePath,
                           &instrumented_image_path_) ||
      instrumented_image_path_.empty()) {
    return Usage(command_line, "Invalid or missing instrumented image path.");
  }

  // Parse the output file path.
  output_file_path_ = command_line->GetSwitchValuePath(kOutputFile);
  if (output_file_path_.empty()) {
    return Usage(command_line, "Invalid or missing output file path.");
  }

  // Parse the (optional) input image path.
  if (!GetDeprecatedSwitch(command_line,
                           kInputImage,
                           kInputDll,
                           &CommandLine::GetSwitchValuePath,
                           &input_image_path_)) {
    return Usage(command_line, "Invalid input image path.");
  }

  bb_entry_count_file_path_ =
      command_line->GetSwitchValuePath(kBasicBlockEntryCounts);

  // Parse the reorderer flags.
  std::string flags_str(command_line->GetSwitchValueASCII(kReordererFlags));
  if (!ParseFlags(flags_str, &flags_)) {
    return Usage(command_line, "Invalid reorderer flags");
  }

  // Parse the pretty-print switch.
  pretty_print_ = command_line->HasSwitch(kPrettyPrint);

  // Make all of the input paths absolute.
  input_image_path_ = AbsolutePath(input_image_path_);
  instrumented_image_path_ = AbsolutePath(instrumented_image_path_);
  output_file_path_ = AbsolutePath(output_file_path_);
  bb_entry_count_file_path_ = AbsolutePath(bb_entry_count_file_path_);

  // Capture the (possibly empty) set of trace files to read.
  for (size_t i = 0; i < command_line->GetArgs().size(); ++i) {
    const base::FilePath pattern(command_line->GetArgs()[i]);
    if (!AppendMatchingPaths(pattern, &trace_file_paths_)) {
      LOG(ERROR) << "Found no files matching '" << pattern.value() << "'.";
      return Usage(command_line, "");
    }
  }

  // Check if we are in random order mode. Look for and parse --seed.
  if (command_line->HasSwitch(kSeed)) {
    if (!trace_file_paths_.empty()) {
      return Usage(command_line,
                   "Trace files are not accepted in random order mode.");
    }
    std::string seed_str(command_line->GetSwitchValueASCII(kSeed));
    int tmp_seed = 0;
    if (seed_str.empty() || !base::StringToInt(seed_str, &tmp_seed)) {
      return Usage(command_line, "Invalid seed value.");
    }
    seed_ = tmp_seed;
    mode_ = kRandomOrderMode;
  }

  // Parse the list-dead-code switch.
  if (command_line->HasSwitch(kListDeadCode)) {
    if (mode_ != kInvalidMode) {
      LOG(ERROR) << "--" << kListDeadCode << " and --" << kSeed << "=N are "
                 << "mutually exclusive.";
      return false;
    }
    mode_ = kDeadCodeFinderMode;
  }

  // If we haven't found anything to over-ride the default mode (linear order),
  // then the default it is.
  if (mode_ == kInvalidMode)
    mode_ = kLinearOrderMode;

  // We do not accept trace file paths in random order mode.
  if (mode_ == kRandomOrderMode && !trace_file_paths_.empty()) {
    return Usage(command_line,
                 "Trace files are not accepted in random order mode.");
  }

  // We only accept a basic-block entry count file in linear order mode, and
  // we require the input image path when we do so.
  if (!bb_entry_count_file_path_.empty()) {
    if (mode_ != kLinearOrderMode) {
      return Usage(command_line,
                   "A basic-block entry counts file is only accepted in linear "
                   "order mode.");
    }
    if (input_image_path_.empty()) {
      return Usage(command_line,
                   "The input image is required for basic-block level "
                   "optimization.");
    }
  }

  // If we get here then the command-line switches were valid.
  return true;
}

bool ReorderApp::SetUp() {
  switch (mode_) {
    case kLinearOrderMode:
      order_generator_.reset(new LinearOrderGenerator());
      return true;

    case kRandomOrderMode:
      order_generator_.reset(new RandomOrderGenerator(seed_));
      return true;

    case kDeadCodeFinderMode:
      order_generator_.reset(new DeadCodeFinder());
      return true;
  }

  NOTREACHED();
  return false;
}

int ReorderApp::Run() {
  pe::PEFile input_image;
  block_graph::BlockGraph block_graph;
  pe::ImageLayout image_layout(&block_graph);
  Reorderer::Order order;
  Reorderer reorderer(input_image_path_,
                      instrumented_image_path_,
                      trace_file_paths_,
                      flags_);

  // Generate a block-level ordering.
  if (!reorderer.Reorder(order_generator_.get(),
                         &order,
                         &input_image,
                         &image_layout)) {
    LOG(ERROR) << "Reorder failed.";
    return 1;
  }

  // Basic-block optimize the resulting order if there is an entry count file.
  if (mode_ == kLinearOrderMode && !bb_entry_count_file_path_.empty()) {
    pe::PEFile::Signature signature;
    input_image.GetSignature(&signature);
    if (!OptimizeBasicBlocks(signature, image_layout, &order)) {
      LOG(ERROR) << "Basic-block optimization failed.";
      return 1;
    }
  }

  // Serialize the order to JSON.
  if (!order.SerializeToJSON(input_image, output_file_path_, pretty_print_)) {
    LOG(ERROR) << "Unable to output order.";
    return 1;
  }

  // We were successful.
  return 0;
}

bool ReorderApp::Usage(const CommandLine* cmd_line,
                       const base::StringPiece& message) const {
  if (!message.empty()) {
    ::fwrite(message.data(), 1, message.length(), err());
    ::fprintf(err(), "\n\n");
  }

  ::fprintf(err(),
            kUsageFormatStr,
            cmd_line->GetProgram().BaseName().value().c_str());

  return false;
}

bool ReorderApp::OptimizeBasicBlocks(const pe::PEFile::Signature& signature,
                                     const pe::ImageLayout& image_layout,
                                     Reorderer::Order* order) {
  DCHECK(order != NULL);

  LOG(INFO) << "Performing basic block ordering.";

  // Load the basic-block entry count data.
  ModuleEntryCountMap module_entry_count_map;
  BasicBlockEntryCountSerializer serializer;
  if (!serializer.LoadFromJson(bb_entry_count_file_path_,
                               &module_entry_count_map)) {
    LOG(ERROR) << "Failed to load basic-block entry count data";
    return false;
  }

  const EntryCountMap* entry_counts = NULL;
  if (!FindEntryCountMap(signature, module_entry_count_map, &entry_counts)) {
    LOG(ERROR) << "Failed to find entry count vector for '"
               << signature.path << "'.";
    return false;
  }

  // Find the PDB file for the module.
  base::FilePath pdb_path;
  if (!pe::FindPdbForModule(instrumented_image_path_, &pdb_path) ||
      pdb_path.empty()) {
    LOG(ERROR) << "Failed to find PDB for instrumented image: "
               << instrumented_image_path_.value();
    return false;
  }

  // Optimize the ordering at the basic-block level.
  BasicBlockOptimizer optimizer;
  if (!optimizer.Optimize(image_layout, *entry_counts, order)) {
    LOG(ERROR) << "Failed to optimize basic-block ordering.";
    return false;
  }

  return true;
}

}  // namespace reorder
<commit_msg>Remove useless dead code in reorder. R=chrisha@chromium.org BUG=<commit_after>// Copyright 2012 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Parses a module and ETW trace files, generating an ordering of the
// blocks in the decomposed image.

#include "syzygy/reorder/reorder_app.h"

#include <objbase.h>
#include <iostream>

#include "base/string_number_conversions.h"
#include "base/stringprintf.h"
#include "base/strings/string_split.h"
#include "syzygy/grinder/basic_block_entry_count_serializer.h"
#include "syzygy/grinder/basic_block_util.h"
#include "syzygy/pe/find.h"
#include "syzygy/reorder/basic_block_optimizer.h"
#include "syzygy/reorder/dead_code_finder.h"
#include "syzygy/reorder/linear_order_generator.h"
#include "syzygy/reorder/random_order_generator.h"

namespace reorder {

namespace {

using grinder::basic_block_util::EntryCountMap;
using grinder::basic_block_util::ModuleEntryCountMap;
using grinder::basic_block_util::LoadBasicBlockRanges;
using grinder::basic_block_util::FindEntryCountMap;
using grinder::basic_block_util::RelativeAddressRangeVector;
using grinder::BasicBlockEntryCountSerializer;

static const char kUsageFormatStr[] =
    "Usage: %ls [options] [log files ...]\n"
    "  Required Options:\n"
    "    --instrumented-image=<path> the path to the instrumented image file.\n"
    "    --output-file=<path> the output file.\n"
    "  Optional Options:\n"
    "    --input-image=<path> the input image file to reorder. If this is not\n"
    "        specified it will be inferred from the instrumented image's\n"
    "        metadata.\n"
    "    --basic-block-entry-counts=PATH the path to the JSON file containing\n"
    "        the summary basic-block entry counts for the image. If this is\n"
    "        given then the input image is also required.\n"
    "    --seed=INT generates a random ordering; don't specify ETW log files.\n"
    "    --list-dead-code instead of an ordering, output the set of functions\n"
    "        not visited during the trace.\n"
    "    --pretty-print enables pretty printing of the JSON output file.\n"
    "    --reorderer-flags=<comma separated reorderer flags>\n"
    "  Reorderer Flags:\n"
    "    no-code: Do not reorder code sections.\n"
    "    no-data: Do not reorder data sections.\n"
    "  Deprecated Options:\n"
    "    --instrumented-dll=<path> aliases to --instrumented-image.\n"
    "    --input-dll=<path> aliases to --input-image.\n";

// Parses reorderer flags. Returns true on success, false otherwise. On
// failure, also outputs Usage with an error message.
bool ParseFlags(const std::string& flags_str, Reorderer::Flags* flags) {
  DCHECK(flags != NULL);

  // Set the default flags value.
  Reorderer::Flags out_flags =
      Reorderer::kFlagReorderData | Reorderer::kFlagReorderCode;

  // If there is a string to process then extract its flags.
  if (!flags_str.empty()) {
    typedef std::vector<std::string> StringVector;
    StringVector text_flags;
    base::SplitString(flags_str, ',', &text_flags);
    StringVector::const_iterator flag_iter = text_flags.begin();
    for (; flag_iter != text_flags.end(); ++flag_iter) {
      if (*flag_iter == "no-data") {
        out_flags &= ~Reorderer::kFlagReorderData;
      } else if (*flag_iter == "no-code") {
        out_flags &= ~Reorderer::kFlagReorderCode;
      } else if (!flag_iter->empty()) {
        LOG(ERROR) << "Unknown reorderer flag: " << *flag_iter << ".";
        return false;
      }
    }
  }

  // Set the return value.
  *flags = out_flags;

  return true;
}

}  // namespace

const char ReorderApp::kInstrumentedImage[] = "instrumented-image";
const char ReorderApp::kOutputFile[] = "output-file";
const char ReorderApp::kInputImage[] = "input-image";
const char ReorderApp::kBasicBlockEntryCounts[] = "basic-block-entry-counts";
const char ReorderApp::kSeed[] = "seed";
const char ReorderApp::kListDeadCode[] = "list-dead-code";
const char ReorderApp::kPrettyPrint[] = "pretty-print";
const char ReorderApp::kReordererFlags[] = "reorderer-flags";
const char ReorderApp::kInstrumentedDll[] = "instrumented-dll";
const char ReorderApp::kInputDll[] = "input-dll";

ReorderApp::ReorderApp()
    : AppImplBase("Reorder"),
      mode_(kInvalidMode),
      seed_(0),
      pretty_print_(false),
      flags_(0) {
}

bool ReorderApp::ParseCommandLine(const CommandLine* command_line) {
  DCHECK(command_line != NULL);
  DCHECK_EQ(kInvalidMode, mode_);

  // Parse the instrumented image path.
  if (!GetDeprecatedSwitch(command_line,
                           kInstrumentedImage,
                           kInstrumentedDll,
                           &CommandLine::GetSwitchValuePath,
                           &instrumented_image_path_) ||
      instrumented_image_path_.empty()) {
    return Usage(command_line, "Invalid or missing instrumented image path.");
  }

  // Parse the output file path.
  output_file_path_ = command_line->GetSwitchValuePath(kOutputFile);
  if (output_file_path_.empty()) {
    return Usage(command_line, "Invalid or missing output file path.");
  }

  // Parse the (optional) input image path.
  if (!GetDeprecatedSwitch(command_line,
                           kInputImage,
                           kInputDll,
                           &CommandLine::GetSwitchValuePath,
                           &input_image_path_)) {
    return Usage(command_line, "Invalid input image path.");
  }

  bb_entry_count_file_path_ =
      command_line->GetSwitchValuePath(kBasicBlockEntryCounts);

  // Parse the reorderer flags.
  std::string flags_str(command_line->GetSwitchValueASCII(kReordererFlags));
  if (!ParseFlags(flags_str, &flags_)) {
    return Usage(command_line, "Invalid reorderer flags");
  }

  // Parse the pretty-print switch.
  pretty_print_ = command_line->HasSwitch(kPrettyPrint);

  // Make all of the input paths absolute.
  input_image_path_ = AbsolutePath(input_image_path_);
  instrumented_image_path_ = AbsolutePath(instrumented_image_path_);
  output_file_path_ = AbsolutePath(output_file_path_);
  bb_entry_count_file_path_ = AbsolutePath(bb_entry_count_file_path_);

  // Capture the (possibly empty) set of trace files to read.
  for (size_t i = 0; i < command_line->GetArgs().size(); ++i) {
    const base::FilePath pattern(command_line->GetArgs()[i]);
    if (!AppendMatchingPaths(pattern, &trace_file_paths_)) {
      LOG(ERROR) << "Found no files matching '" << pattern.value() << "'.";
      return Usage(command_line, "");
    }
  }

  // Check if we are in random order mode. Look for and parse --seed.
  if (command_line->HasSwitch(kSeed)) {
    if (!trace_file_paths_.empty()) {
      return Usage(command_line,
                   "Trace files are not accepted in random order mode.");
    }
    std::string seed_str(command_line->GetSwitchValueASCII(kSeed));
    int tmp_seed = 0;
    if (seed_str.empty() || !base::StringToInt(seed_str, &tmp_seed)) {
      return Usage(command_line, "Invalid seed value.");
    }
    seed_ = tmp_seed;
    mode_ = kRandomOrderMode;
  }

  // Parse the list-dead-code switch.
  if (command_line->HasSwitch(kListDeadCode)) {
    if (mode_ != kInvalidMode) {
      LOG(ERROR) << "--" << kListDeadCode << " and --" << kSeed << "=N are "
                 << "mutually exclusive.";
      return false;
    }
    mode_ = kDeadCodeFinderMode;
  }

  // If we haven't found anything to over-ride the default mode (linear order),
  // then the default it is.
  if (mode_ == kInvalidMode)
    mode_ = kLinearOrderMode;

  // We do not accept trace file paths in random order mode.
  if (mode_ == kRandomOrderMode && !trace_file_paths_.empty()) {
    return Usage(command_line,
                 "Trace files are not accepted in random order mode.");
  }

  // We only accept a basic-block entry count file in linear order mode, and
  // we require the input image path when we do so.
  if (!bb_entry_count_file_path_.empty()) {
    if (mode_ != kLinearOrderMode) {
      return Usage(command_line,
                   "A basic-block entry counts file is only accepted in linear "
                   "order mode.");
    }
    if (input_image_path_.empty()) {
      return Usage(command_line,
                   "The input image is required for basic-block level "
                   "optimization.");
    }
  }

  // If we get here then the command-line switches were valid.
  return true;
}

bool ReorderApp::SetUp() {
  switch (mode_) {
    case kLinearOrderMode:
      order_generator_.reset(new LinearOrderGenerator());
      return true;

    case kRandomOrderMode:
      order_generator_.reset(new RandomOrderGenerator(seed_));
      return true;

    case kDeadCodeFinderMode:
      order_generator_.reset(new DeadCodeFinder());
      return true;
  }

  NOTREACHED();
  return false;
}

int ReorderApp::Run() {
  pe::PEFile input_image;
  block_graph::BlockGraph block_graph;
  pe::ImageLayout image_layout(&block_graph);
  Reorderer::Order order;
  Reorderer reorderer(input_image_path_,
                      instrumented_image_path_,
                      trace_file_paths_,
                      flags_);

  // Generate a block-level ordering.
  if (!reorderer.Reorder(order_generator_.get(),
                         &order,
                         &input_image,
                         &image_layout)) {
    LOG(ERROR) << "Reorder failed.";
    return 1;
  }

  // Basic-block optimize the resulting order if there is an entry count file.
  if (mode_ == kLinearOrderMode && !bb_entry_count_file_path_.empty()) {
    pe::PEFile::Signature signature;
    input_image.GetSignature(&signature);
    if (!OptimizeBasicBlocks(signature, image_layout, &order)) {
      LOG(ERROR) << "Basic-block optimization failed.";
      return 1;
    }
  }

  // Serialize the order to JSON.
  if (!order.SerializeToJSON(input_image, output_file_path_, pretty_print_)) {
    LOG(ERROR) << "Unable to output order.";
    return 1;
  }

  // We were successful.
  return 0;
}

bool ReorderApp::Usage(const CommandLine* cmd_line,
                       const base::StringPiece& message) const {
  if (!message.empty()) {
    ::fwrite(message.data(), 1, message.length(), err());
    ::fprintf(err(), "\n\n");
  }

  ::fprintf(err(),
            kUsageFormatStr,
            cmd_line->GetProgram().BaseName().value().c_str());

  return false;
}

bool ReorderApp::OptimizeBasicBlocks(const pe::PEFile::Signature& signature,
                                     const pe::ImageLayout& image_layout,
                                     Reorderer::Order* order) {
  DCHECK(order != NULL);

  LOG(INFO) << "Performing basic block ordering.";

  // Load the basic-block entry count data.
  ModuleEntryCountMap module_entry_count_map;
  BasicBlockEntryCountSerializer serializer;
  if (!serializer.LoadFromJson(bb_entry_count_file_path_,
                               &module_entry_count_map)) {
    LOG(ERROR) << "Failed to load basic-block entry count data";
    return false;
  }

  const EntryCountMap* entry_counts = NULL;
  if (!FindEntryCountMap(signature, module_entry_count_map, &entry_counts)) {
    LOG(ERROR) << "Failed to find entry count vector for '"
               << signature.path << "'.";
    return false;
  }

  // Optimize the ordering at the basic-block level.
  BasicBlockOptimizer optimizer;
  if (!optimizer.Optimize(image_layout, *entry_counts, order)) {
    LOG(ERROR) << "Failed to optimize basic-block ordering.";
    return false;
  }

  return true;
}

}  // namespace reorder
<|endoftext|>


<commit_before>
#ifndef OMP_OMP_HPP
#define OMP_OMP_HPP

#include <cstdint>
#include <functional>
#include <thread>
#include <vector>
#include <mutex>
#include <condition_variable>
#include <cassert>
#include <iterator>

namespace omp
{

  struct iteration_context
  {
    std::size_t thread_index;
    std::size_t index;
  };

  namespace internal
  {
    extern std::mutex global_mutex;
    extern const unsigned default_num_threads;

    class thread_pool
    {
    public:
      thread_pool(const std::function<void(std::size_t thread_idx)>& fn, unsigned num_threads = 0) :
        num_threads_(num_threads ? num_threads : default_num_threads)
      {
        threads_.reserve(num_threads - 1);
        for (unsigned i = 0; i < (num_threads_ - 1); ++i)
          threads_.emplace_back(fn, i);
        fn(num_threads_ - 1);
        for (auto it = threads_.begin(); it != threads_.end(); ++it)
          it->join();
      }
    private:
      const std::size_t num_threads_;
      std::vector<std::thread> threads_;
    };

    class thread_pool2
    {
    private:
      enum class state { shutdown = 0, run, running, sleep };
      std::vector<std::thread> threads_;
      std::vector<state> states_;
      std::mutex mtx_;
      std::condition_variable cv_;
      std::function<void(std::size_t)> fn_;
    public:
      thread_pool2(std::size_t num_threads = 0) :
        states_(num_threads ? num_threads - 1 : default_num_threads - 1, state::sleep)
      {
        threads_.reserve(states_.size());
        for (std::size_t i = 0; i < states_.size(); ++i)
        {
          threads_.emplace_back(std::bind(&thread_pool2::routine, this, i));
        }
      }

      ~thread_pool2()
      {
        {
          std::unique_lock<std::mutex> lk(mtx_);
          std::fill(states_.begin(), states_.end(), state::shutdown);
        }

        cv_.notify_all();

        for (auto& t : threads_)
          t.join();
      }

      std::size_t thread_count() const { return threads_.size() + 1; }

      void routine(std::size_t thread_idx)
      {
        while (true)
        {
          {
            std::unique_lock<std::mutex> lk(mtx_);
            if (states_[thread_idx] == state::shutdown)
              break;
            if (states_[thread_idx] == state::running)
            {
              states_[thread_idx] = state::sleep;
              cv_.notify_all();
            }
            cv_.wait(lk, [this, thread_idx] { return states_[thread_idx] != state::sleep; });
            if (states_[thread_idx] == state::shutdown)
              break;
            states_[thread_idx] = state::running;
          }

          if (fn_)
          {
            fn_(thread_idx);
          }
        }
      }

      //template <typename Fn>
      void operator()(std::function<void(std::size_t)>&& fn)
      {
        fn_ = std::move(fn);

        {
          std::unique_lock<std::mutex> lk(mtx_);
          std::fill(states_.begin(), states_.end(), state::run);
        }
        cv_.notify_all();

        if (fn_)
        {
          fn_(states_.size());
        }

        {
          // Wait for child threads to complete.
          std::unique_lock<std::mutex> lk(mtx_);
          if (std::count(states_.begin(), states_.end(), state::sleep) != states_.size()) // If I remember correctly, this check is redundant.
          {
            cv_.wait(lk, [this] { return std::count(states_.begin(), states_.end(), state::sleep) == states_.size(); });
          }
        }

        fn_ = nullptr;
      }
    };

    template<typename Iter>
    class dynamic_iterator_thread_pool
    {
    public:
      dynamic_iterator_thread_pool(std::size_t chunk_size, Iter begin, Iter end, const std::function<void(typename Iter::reference, const iteration_context&)>& fn, unsigned num_threads) :
        fn_(fn),
        cur_(begin),
        end_(end),
        index_(0),
        chunk_size_(chunk_size ? chunk_size : 1),
        num_threads_(num_threads ? num_threads : default_num_threads)
      {
        threads_.reserve(num_threads_ - 1);
        for (unsigned i = 0; i < (num_threads_ - 1); ++i)
          threads_.emplace_back(std::bind(&dynamic_iterator_thread_pool::routine, this, i));
        this->routine(num_threads_ - 1);

        for (auto it = threads_.begin(); it != threads_.end(); ++it)
          it->join();
      }

    private:
      std::function<void(typename Iter::reference, const omp::iteration_context&)> fn_;
      std::vector<std::thread> threads_;
      Iter cur_;
      const Iter end_;
      std::size_t index_;
      std::mutex mtx_;
      const std::size_t chunk_size_;
      const std::size_t num_threads_;

      void routine(std::size_t thread_index)
      {
        bool done = false;
        while (!done)
        {
          std::vector<Iter> chunk(chunk_size_);

          std::unique_lock<std::mutex> lk(mtx_);
          std::size_t index = index_;
          for (std::size_t chunk_offset = 0; chunk_offset < chunk.size(); ++chunk_offset)
          {
            ++index_;
            chunk[chunk_offset] = cur_;
            if (cur_ != end_)
              ++cur_;
          }
          lk.unlock();

          for (std::size_t chunk_offset = 0; chunk_offset < chunk.size(); ++chunk_offset)
          {
            if (chunk[chunk_offset] == end_)
            {
              done = true;
            }
            else
            {
              fn_(*chunk[chunk_offset], {thread_index, index + chunk_offset}); //fn_ ? fn_(*it, i) : void();
            }
          }
        }
      }
    };

    template<typename Iter>
    class static_iterator_thread_pool
    {
    public:
      static_iterator_thread_pool(std::size_t chunk_size, Iter begin, Iter end, const std::function<void(typename Iter::reference,const iteration_context&)>& fn, unsigned num_threads = 0) :
        fn_(fn),
        num_threads_(num_threads ? num_threads : default_num_threads),
        beg_(begin),
        end_(end),
        total_elements_(std::distance(beg_, end_)),
        chunk_size_(chunk_size ? chunk_size : static_cast<std::size_t>(total_elements_) / num_threads_)
      {
        threads_.reserve(num_threads_ - 1);
        for (unsigned i = 0; i < (num_threads_ - 1); ++i)
          threads_.emplace_back(std::bind(&static_iterator_thread_pool::routine, this, i));
        this->routine(num_threads_ - 1);

        for (auto it = threads_.begin(); it != threads_.end(); ++it)
          it->join();
      }
    private:
      std::function<void(typename Iter::reference, const omp::iteration_context&)> fn_;
      const std::size_t num_threads_;
      std::vector<std::thread> threads_;
      const Iter beg_;
      const Iter end_;
      long total_elements_;
      const std::size_t chunk_size_;
    public:
      void routine(std::size_t thread_index)
      {
        auto cur = beg_;

        std::advance(cur, thread_index * chunk_size_);
        for (std::size_t index = (thread_index * chunk_size_); index < total_elements_; index += (chunk_size_ * num_threads_ - chunk_size_), std::advance(cur, chunk_size_ * num_threads_ - chunk_size_))
        {
          for (std::size_t chunk_offset = 0; chunk_offset < chunk_size_ && index < total_elements_; ++chunk_offset)
          {
            assert(cur != end_);
            fn_(*cur, {thread_index,index}); //fn_ ? fn_(*it, i) : void();
            ++cur;
            ++index;
          }
        }
      }
    };

    template<typename Iter>
    class static_iterator_functor
    {
    public:
      static_iterator_functor(std::size_t chunk_size, Iter begin, Iter end, const std::function<void(typename Iter::reference,const iteration_context&)>& fn, unsigned num_threads) :
        fn_(fn),
        num_threads_(num_threads ? num_threads : default_num_threads),
        beg_(begin),
        end_(end),
        total_elements_(std::distance(beg_, end_)),
        chunk_size_(chunk_size ? chunk_size : static_cast<std::size_t>(total_elements_) / num_threads_)
      {
//        threads_.reserve(num_threads_ - 1);
//        for (unsigned i = 0; i < (num_threads_ - 1); ++i)
//          threads_.emplace_back(std::bind(&static_iterator_thread_pool::routine, this, i));
//        this->routine(num_threads_ - 1);
//
//        for (auto it = threads_.begin(); it != threads_.end(); ++it)
//          it->join();
      }
    private:
      std::function<void(typename Iter::reference, const omp::iteration_context&)> fn_;
      const std::size_t num_threads_;
      const Iter beg_;
      const Iter end_;
      long total_elements_;
      const std::size_t chunk_size_;
    public:
      void operator()(std::size_t thread_index)
      {
        auto cur = beg_;

        std::advance(cur, thread_index * chunk_size_);
        for (std::size_t index = (thread_index * chunk_size_); index < total_elements_; index += (chunk_size_ * num_threads_ - chunk_size_), std::advance(cur, chunk_size_ * num_threads_ - chunk_size_))
        {
          for (std::size_t chunk_offset = 0; chunk_offset < chunk_size_ && index < total_elements_; ++chunk_offset)
          {
            assert(cur != end_);
            fn_(*cur, {thread_index,index}); //fn_ ? fn_(*it, i) : void();
            ++cur;
            ++index;
          }
        }
      }
    };
  }

  class sequence_iterator
  {
  public:
    typedef sequence_iterator self_type;
    typedef int difference_type;
    typedef int value_type;
    typedef value_type& reference;
    typedef value_type* pointer;
    typedef std::random_access_iterator_tag iterator_category;

    sequence_iterator() : val_(0) {}
    sequence_iterator(value_type val) :
      val_(val)
    {

    }

    //reference          operator [] (difference_type);
    bool operator < (const self_type& other) { return val_ < other.val_; }
    bool operator > (const self_type& other) { return val_ > other.val_; }
    bool operator <= (const self_type& other) { return val_ <= other.val_; }
    bool operator >= (const self_type& other) { return val_ >= other.val_; }

    self_type operator++()
    {
      self_type ret = *this;
      ++val_;
      return ret;
    }

    self_type operator--()
    {
      self_type ret = *this;
      --val_;
      return ret;
    }

    self_type&      operator += (difference_type i)  { val_ += i; return *this; }
    self_type&      operator -= (difference_type i)  { val_ -= i; return *this; }
    self_type       operator +  (difference_type i) { return self_type(val_ + i); }
    self_type       operator -  (difference_type i) { return self_type(val_ - i); }
    difference_type operator - (const self_type& other) { return val_ - other.val_; }

    void operator++(int) { ++val_; }
    void operator--(int) { --val_; }
    reference operator*() { return val_; }
    pointer operator->() { return &val_; }
    bool operator==(const self_type& rhs) const { return (val_ == rhs.val_); }
    bool operator!=(const self_type& rhs) const { return (val_ != rhs.val_); }
  private:
    value_type val_;
  };

  class schedule
  {
  public:
    schedule(std::size_t chunk_size);
    std::size_t chunk_size() const;
  protected:
    std::size_t chunk_size_;
  };

  class dynamic_schedule : public schedule
  {
  public:
    dynamic_schedule(std::size_t chunk_size = 0);
  };

  class static_schedule : public schedule
  {
  public:
    static_schedule(std::size_t chunk_size = 0);
  };

  void parallel(const std::function<void(std::size_t)>& operation, unsigned thread_cnt = 0);

  template <typename Iterator>
  void parallel_for(const dynamic_schedule& sched, Iterator begin, Iterator end, const std::function<void(typename Iterator::reference, const iteration_context&)>& operation, unsigned thread_cnt = 0)
  {
    internal::dynamic_iterator_thread_pool<Iterator> pool(sched.chunk_size(), begin, end, operation, thread_cnt);
  }

  template <typename Iterator>
  void parallel_for(const static_schedule& sched, Iterator begin, Iterator end, const std::function<void(typename Iterator::reference, const iteration_context&)>& operation, unsigned thread_cnt = 0)
  {
    internal::static_iterator_thread_pool<Iterator> pool(sched.chunk_size(), begin, end, operation, thread_cnt);
  }

  template <typename Iterator>
  void parallel_for_exp(const static_schedule& sched, Iterator begin, Iterator end, const std::function<void(typename Iterator::reference, const iteration_context&)>& operation, internal::thread_pool2& tp)
  {

    tp(internal::static_iterator_functor<Iterator>(sched.chunk_size(), begin, end, operation, tp.thread_count())); //std::bind(&internal::static_iterator_functor<Iterator>::routine, &static_fn, std::placeholders::_1));
  }

  template <typename Iterator>
  void parallel_for(Iterator begin, Iterator end, const std::function<void(typename Iterator::reference, const iteration_context&)>& operation, unsigned thread_cnt = 0)
  {
    parallel_for(static_schedule(), begin, end, operation, thread_cnt);
  }

  template <typename Handler>
  void critical(std::mutex& mtx, Handler fn)
  {
    std::lock_guard<std::mutex> lk(mtx);
    fn();
  }

  template <typename Handler>
  void critical(Handler fn)
  {
    std::lock_guard<std::mutex> lk(internal::global_mutex);
    fn();
  }
}

#endif //OMP_OMP_HPP
<commit_msg>Fixes build error on linux.<commit_after>
#ifndef OMP_OMP_HPP
#define OMP_OMP_HPP

#include <cstdint>
#include <functional>
#include <thread>
#include <vector>
#include <mutex>
#include <condition_variable>
#include <cassert>
#include <iterator>
#include <algorithm>

namespace omp
{

  struct iteration_context
  {
    std::size_t thread_index;
    std::size_t index;
  };

  namespace internal
  {
    extern std::mutex global_mutex;
    extern const unsigned default_num_threads;

    class thread_pool
    {
    public:
      thread_pool(const std::function<void(std::size_t thread_idx)>& fn, unsigned num_threads = 0) :
        num_threads_(num_threads ? num_threads : default_num_threads)
      {
        threads_.reserve(num_threads - 1);
        for (unsigned i = 0; i < (num_threads_ - 1); ++i)
          threads_.emplace_back(fn, i);
        fn(num_threads_ - 1);
        for (auto it = threads_.begin(); it != threads_.end(); ++it)
          it->join();
      }
    private:
      const std::size_t num_threads_;
      std::vector<std::thread> threads_;
    };

    class thread_pool2
    {
    private:
      enum class state { shutdown = 0, run, running, sleep };
      std::vector<std::thread> threads_;
      std::vector<state> states_;
      std::mutex mtx_;
      std::condition_variable cv_;
      std::function<void(std::size_t)> fn_;
    public:
      thread_pool2(std::size_t num_threads = 0) :
        states_(num_threads ? num_threads - 1 : default_num_threads - 1, state::sleep)
      {
        threads_.reserve(states_.size());
        for (std::size_t i = 0; i < states_.size(); ++i)
        {
          threads_.emplace_back(std::bind(&thread_pool2::routine, this, i));
        }
      }

      ~thread_pool2()
      {
        {
          std::unique_lock<std::mutex> lk(mtx_);
          std::fill(states_.begin(), states_.end(), state::shutdown);
        }

        cv_.notify_all();

        for (auto& t : threads_)
          t.join();
      }

      std::size_t thread_count() const { return threads_.size() + 1; }

      void routine(std::size_t thread_idx)
      {
        while (true)
        {
          {
            std::unique_lock<std::mutex> lk(mtx_);
            if (states_[thread_idx] == state::shutdown)
              break;
            if (states_[thread_idx] == state::running)
            {
              states_[thread_idx] = state::sleep;
              cv_.notify_all();
            }
            cv_.wait(lk, [this, thread_idx] { return states_[thread_idx] != state::sleep; });
            if (states_[thread_idx] == state::shutdown)
              break;
            states_[thread_idx] = state::running;
          }

          if (fn_)
          {
            fn_(thread_idx);
          }
        }
      }

      //template <typename Fn>
      void operator()(std::function<void(std::size_t)>&& fn)
      {
        fn_ = std::move(fn);

        {
          std::unique_lock<std::mutex> lk(mtx_);
          std::fill(states_.begin(), states_.end(), state::run);
        }
        cv_.notify_all();

        if (fn_)
        {
          fn_(states_.size());
        }

        {
          // Wait for child threads to complete.
          std::unique_lock<std::mutex> lk(mtx_);
          if (std::count(states_.begin(), states_.end(), state::sleep) != states_.size()) // If I remember correctly, this check is redundant.
          {
            cv_.wait(lk, [this] { return std::count(states_.begin(), states_.end(), state::sleep) == states_.size(); });
          }
        }

        fn_ = nullptr;
      }
    };

    template<typename Iter>
    class dynamic_iterator_thread_pool
    {
    public:
      dynamic_iterator_thread_pool(std::size_t chunk_size, Iter begin, Iter end, const std::function<void(typename Iter::reference, const iteration_context&)>& fn, unsigned num_threads) :
        fn_(fn),
        cur_(begin),
        end_(end),
        index_(0),
        chunk_size_(chunk_size ? chunk_size : 1),
        num_threads_(num_threads ? num_threads : default_num_threads)
      {
        threads_.reserve(num_threads_ - 1);
        for (unsigned i = 0; i < (num_threads_ - 1); ++i)
          threads_.emplace_back(std::bind(&dynamic_iterator_thread_pool::routine, this, i));
        this->routine(num_threads_ - 1);

        for (auto it = threads_.begin(); it != threads_.end(); ++it)
          it->join();
      }

    private:
      std::function<void(typename Iter::reference, const omp::iteration_context&)> fn_;
      std::vector<std::thread> threads_;
      Iter cur_;
      const Iter end_;
      std::size_t index_;
      std::mutex mtx_;
      const std::size_t chunk_size_;
      const std::size_t num_threads_;

      void routine(std::size_t thread_index)
      {
        bool done = false;
        while (!done)
        {
          std::vector<Iter> chunk(chunk_size_);

          std::unique_lock<std::mutex> lk(mtx_);
          std::size_t index = index_;
          for (std::size_t chunk_offset = 0; chunk_offset < chunk.size(); ++chunk_offset)
          {
            ++index_;
            chunk[chunk_offset] = cur_;
            if (cur_ != end_)
              ++cur_;
          }
          lk.unlock();

          for (std::size_t chunk_offset = 0; chunk_offset < chunk.size(); ++chunk_offset)
          {
            if (chunk[chunk_offset] == end_)
            {
              done = true;
            }
            else
            {
              fn_(*chunk[chunk_offset], {thread_index, index + chunk_offset}); //fn_ ? fn_(*it, i) : void();
            }
          }
        }
      }
    };

    template<typename Iter>
    class static_iterator_thread_pool
    {
    public:
      static_iterator_thread_pool(std::size_t chunk_size, Iter begin, Iter end, const std::function<void(typename Iter::reference,const iteration_context&)>& fn, unsigned num_threads = 0) :
        fn_(fn),
        num_threads_(num_threads ? num_threads : default_num_threads),
        beg_(begin),
        end_(end),
        total_elements_(std::distance(beg_, end_)),
        chunk_size_(chunk_size ? chunk_size : static_cast<std::size_t>(total_elements_) / num_threads_)
      {
        threads_.reserve(num_threads_ - 1);
        for (unsigned i = 0; i < (num_threads_ - 1); ++i)
          threads_.emplace_back(std::bind(&static_iterator_thread_pool::routine, this, i));
        this->routine(num_threads_ - 1);

        for (auto it = threads_.begin(); it != threads_.end(); ++it)
          it->join();
      }
    private:
      std::function<void(typename Iter::reference, const omp::iteration_context&)> fn_;
      const std::size_t num_threads_;
      std::vector<std::thread> threads_;
      const Iter beg_;
      const Iter end_;
      long total_elements_;
      const std::size_t chunk_size_;
    public:
      void routine(std::size_t thread_index)
      {
        auto cur = beg_;

        std::advance(cur, thread_index * chunk_size_);
        for (std::size_t index = (thread_index * chunk_size_); index < total_elements_; index += (chunk_size_ * num_threads_ - chunk_size_), std::advance(cur, chunk_size_ * num_threads_ - chunk_size_))
        {
          for (std::size_t chunk_offset = 0; chunk_offset < chunk_size_ && index < total_elements_; ++chunk_offset)
          {
            assert(cur != end_);
            fn_(*cur, {thread_index,index}); //fn_ ? fn_(*it, i) : void();
            ++cur;
            ++index;
          }
        }
      }
    };

    template<typename Iter>
    class static_iterator_functor
    {
    public:
      static_iterator_functor(std::size_t chunk_size, Iter begin, Iter end, const std::function<void(typename Iter::reference,const iteration_context&)>& fn, unsigned num_threads) :
        fn_(fn),
        num_threads_(num_threads ? num_threads : default_num_threads),
        beg_(begin),
        end_(end),
        total_elements_(std::distance(beg_, end_)),
        chunk_size_(chunk_size ? chunk_size : static_cast<std::size_t>(total_elements_) / num_threads_)
      {
//        threads_.reserve(num_threads_ - 1);
//        for (unsigned i = 0; i < (num_threads_ - 1); ++i)
//          threads_.emplace_back(std::bind(&static_iterator_thread_pool::routine, this, i));
//        this->routine(num_threads_ - 1);
//
//        for (auto it = threads_.begin(); it != threads_.end(); ++it)
//          it->join();
      }
    private:
      std::function<void(typename Iter::reference, const omp::iteration_context&)> fn_;
      const std::size_t num_threads_;
      const Iter beg_;
      const Iter end_;
      long total_elements_;
      const std::size_t chunk_size_;
    public:
      void operator()(std::size_t thread_index)
      {
        auto cur = beg_;

        std::advance(cur, thread_index * chunk_size_);
        for (std::size_t index = (thread_index * chunk_size_); index < total_elements_; index += (chunk_size_ * num_threads_ - chunk_size_), std::advance(cur, chunk_size_ * num_threads_ - chunk_size_))
        {
          for (std::size_t chunk_offset = 0; chunk_offset < chunk_size_ && index < total_elements_; ++chunk_offset)
          {
            assert(cur != end_);
            fn_(*cur, {thread_index,index}); //fn_ ? fn_(*it, i) : void();
            ++cur;
            ++index;
          }
        }
      }
    };
  }

  class sequence_iterator
  {
  public:
    typedef sequence_iterator self_type;
    typedef int difference_type;
    typedef int value_type;
    typedef value_type& reference;
    typedef value_type* pointer;
    typedef std::random_access_iterator_tag iterator_category;

    sequence_iterator() : val_(0) {}
    sequence_iterator(value_type val) :
      val_(val)
    {

    }

    //reference          operator [] (difference_type);
    bool operator < (const self_type& other) { return val_ < other.val_; }
    bool operator > (const self_type& other) { return val_ > other.val_; }
    bool operator <= (const self_type& other) { return val_ <= other.val_; }
    bool operator >= (const self_type& other) { return val_ >= other.val_; }

    self_type operator++()
    {
      self_type ret = *this;
      ++val_;
      return ret;
    }

    self_type operator--()
    {
      self_type ret = *this;
      --val_;
      return ret;
    }

    self_type&      operator += (difference_type i)  { val_ += i; return *this; }
    self_type&      operator -= (difference_type i)  { val_ -= i; return *this; }
    self_type       operator +  (difference_type i) { return self_type(val_ + i); }
    self_type       operator -  (difference_type i) { return self_type(val_ - i); }
    difference_type operator - (const self_type& other) { return val_ - other.val_; }

    void operator++(int) { ++val_; }
    void operator--(int) { --val_; }
    reference operator*() { return val_; }
    pointer operator->() { return &val_; }
    bool operator==(const self_type& rhs) const { return (val_ == rhs.val_); }
    bool operator!=(const self_type& rhs) const { return (val_ != rhs.val_); }
  private:
    value_type val_;
  };

  class schedule
  {
  public:
    schedule(std::size_t chunk_size);
    std::size_t chunk_size() const;
  protected:
    std::size_t chunk_size_;
  };

  class dynamic_schedule : public schedule
  {
  public:
    dynamic_schedule(std::size_t chunk_size = 0);
  };

  class static_schedule : public schedule
  {
  public:
    static_schedule(std::size_t chunk_size = 0);
  };

  void parallel(const std::function<void(std::size_t)>& operation, unsigned thread_cnt = 0);

  template <typename Iterator>
  void parallel_for(const dynamic_schedule& sched, Iterator begin, Iterator end, const std::function<void(typename Iterator::reference, const iteration_context&)>& operation, unsigned thread_cnt = 0)
  {
    internal::dynamic_iterator_thread_pool<Iterator> pool(sched.chunk_size(), begin, end, operation, thread_cnt);
  }

  template <typename Iterator>
  void parallel_for(const static_schedule& sched, Iterator begin, Iterator end, const std::function<void(typename Iterator::reference, const iteration_context&)>& operation, unsigned thread_cnt = 0)
  {
    internal::static_iterator_thread_pool<Iterator> pool(sched.chunk_size(), begin, end, operation, thread_cnt);
  }

  template <typename Iterator>
  void parallel_for_exp(const static_schedule& sched, Iterator begin, Iterator end, const std::function<void(typename Iterator::reference, const iteration_context&)>& operation, internal::thread_pool2& tp)
  {

    tp(internal::static_iterator_functor<Iterator>(sched.chunk_size(), begin, end, operation, tp.thread_count())); //std::bind(&internal::static_iterator_functor<Iterator>::routine, &static_fn, std::placeholders::_1));
  }

  template <typename Iterator>
  void parallel_for(Iterator begin, Iterator end, const std::function<void(typename Iterator::reference, const iteration_context&)>& operation, unsigned thread_cnt = 0)
  {
    parallel_for(static_schedule(), begin, end, operation, thread_cnt);
  }

  template <typename Handler>
  void critical(std::mutex& mtx, Handler fn)
  {
    std::lock_guard<std::mutex> lk(mtx);
    fn();
  }

  template <typename Handler>
  void critical(Handler fn)
  {
    std::lock_guard<std::mutex> lk(internal::global_mutex);
    fn();
  }
}

#endif //OMP_OMP_HPP
<|endoftext|>


<commit_before>#ifndef UMINTL_EVALUATION_POLICY_HPP
#define UMINTL_EVALUATION_POLICY_HPP

#include <cstddef>
#include "umintl/forwards.h"
#include "umintl/optimization_context.hpp"
#include <cmath>

namespace umintl{

/** @brief The model_base class
 *
 *  The optimization model can be either deterministic or stochastic.  The latter usually corresponds to expected losses
 * evaluated accross a large amount of data points
 */
template<class BackendType>
struct model_base{
    virtual ~model_base(){ }
    virtual bool update(optimization_context<BackendType> & context) = 0;
    virtual value_gradient get_value_gradient_tag() const = 0;
    virtual hessian_vector_product get_hv_product_tag() const = 0;
};

/** @brief The deterministic class
 *
 *  Assumes the function evaluation is the same at each call. In the case of expected losses, it means all the data-points
 * are always used.
 */
template<class BackendType>
struct deterministic : public model_base<BackendType> {
    bool update(optimization_context<BackendType> &){ return false; }
    value_gradient get_value_gradient_tag() const { return value_gradient(DETERMINISTIC,0,0); }
    hessian_vector_product get_hv_product_tag() const { return hessian_vector_product(DETERMINISTIC,0,0); }
};

template<class BackendType>
struct mini_batch : public model_base<BackendType> {
  public:
    mini_batch(size_t sample_size, size_t dataset_size) : sample_size_(std::min(sample_size,dataset_size)), offset_(0), dataset_size_(dataset_size){ }
    bool update(optimization_context<BackendType> &){
      offset_=(offset_+sample_size_)%dataset_size_;
      return false;
    }
    value_gradient get_value_gradient_tag() const { return value_gradient(STOCHASTIC,dataset_size_,0); }
    hessian_vector_product get_hv_product_tag() const { return hessian_vector_product(STOCHASTIC,sample_size_,offset_); }
private:
    size_t sample_size_;
    size_t offset_;
    size_t dataset_size_;
};

/** @brief the dynamically_sampled class
 *
 * Uses the dynamic sampled procedure from Byrd et al. (2012) :
 * "Sample Size Selection in Optimization Methods for Machine Learning"
 * Requires that the functor overloads :
 * void operator()(VectorType const & X, VectorType & variance, umintl::gradient_variance_tag tag)
 *
 * The parameter tag contains the information on the current offset and sample size
 */
template<class BackendType>
struct dynamically_sampled : public model_base<BackendType> {
  private:
    typedef typename BackendType::ScalarType ScalarType;
    typedef typename BackendType::VectorType VectorType;

  public:
    dynamically_sampled(double r, size_t fbatch, size_t dataset_size, double theta = 0.5) : theta_(theta), r_(r), S(std::min(fbatch,dataset_size)), offset_(0), H_offset_(0), N(dataset_size){ }

    bool update(optimization_context<BackendType> & c){
      if(S==N){
        H_offset_=(H_offset_+(int)(r_*S))%(S - (int)(r_*S) + 1);
        return false;
      }
      else{
        VectorType var = BackendType::create_vector(c.N());
        c.fun().compute_gradient_variance(c.x(),var,gradient_variance(STOCHASTIC,S,offset_));

        //is_descent_direction = norm1(var)/S*[(N-S)/(N-1)] <= theta^2*norm2(grad)^2
        ScalarType nrm1var = BackendType::asum(c.N(),var);
        ScalarType nrm2grad = BackendType::nrm2(c.N(),c.g());
        //std::gradient_variance << nrm1var*scal << " " << std::pow(theta_,2)*std::pow(nrm2grad,2) << std::endl;
        bool is_descent_direction = (nrm1var/S <= (std::pow(theta_,2)*std::pow(nrm2grad,2)));

        //Update parameters
        //size_t old_S = S;
        if(is_descent_direction==false){
          S = (size_t)(nrm1var/std::pow(theta_*nrm2grad,2));
          S = std::min(S,N);
          if(S>N/2)
            S=N;
        }
        offset_=(offset_+S)%(N-S+1);

        if(is_descent_direction==false)
          H_offset_ = 0;
        else
          H_offset_=(H_offset_+S)%(S - (int)(r_*S) + 1);

        BackendType::delete_if_dynamically_allocated(var);
        return true;
      }
    }

    value_gradient get_value_gradient_tag() const {
      return value_gradient(STOCHASTIC,S,offset_);
    }

    hessian_vector_product get_hv_product_tag() const {
      return hessian_vector_product(STOCHASTIC, (size_t)r_*S, H_offset_+offset_);
    }
private:
    double theta_;
    double r_;
    size_t S;
    size_t offset_;
    size_t H_offset_;
    size_t N;
};

}
#endif
<commit_msg>Optimization: Fixed invalid cast<commit_after>#ifndef UMINTL_EVALUATION_POLICY_HPP
#define UMINTL_EVALUATION_POLICY_HPP

#include <cstddef>
#include "umintl/forwards.h"
#include "umintl/optimization_context.hpp"
#include <cmath>

namespace umintl{

/** @brief The model_base class
 *
 *  The optimization model can be either deterministic or stochastic.  The latter usually corresponds to expected losses
 * evaluated accross a large amount of data points
 */
template<class BackendType>
struct model_base{
    virtual ~model_base(){ }
    virtual bool update(optimization_context<BackendType> & context) = 0;
    virtual value_gradient get_value_gradient_tag() const = 0;
    virtual hessian_vector_product get_hv_product_tag() const = 0;
};

/** @brief The deterministic class
 *
 *  Assumes the function evaluation is the same at each call. In the case of expected losses, it means all the data-points
 * are always used.
 */
template<class BackendType>
struct deterministic : public model_base<BackendType> {
    bool update(optimization_context<BackendType> &){ return false; }
    value_gradient get_value_gradient_tag() const { return value_gradient(DETERMINISTIC,0,0); }
    hessian_vector_product get_hv_product_tag() const { return hessian_vector_product(DETERMINISTIC,0,0); }
};

template<class BackendType>
struct mini_batch : public model_base<BackendType> {
  public:
    mini_batch(size_t sample_size, size_t dataset_size) : sample_size_(std::min(sample_size,dataset_size)), offset_(0), dataset_size_(dataset_size){ }
    bool update(optimization_context<BackendType> &){
      offset_=(offset_+sample_size_)%dataset_size_;
      return false;
    }
    value_gradient get_value_gradient_tag() const { return value_gradient(STOCHASTIC,dataset_size_,0); }
    hessian_vector_product get_hv_product_tag() const { return hessian_vector_product(STOCHASTIC,sample_size_,offset_); }
private:
    size_t sample_size_;
    size_t offset_;
    size_t dataset_size_;
};

/** @brief the dynamically_sampled class
 *
 * Uses the dynamic sampled procedure from Byrd et al. (2012) :
 * "Sample Size Selection in Optimization Methods for Machine Learning"
 * Requires that the functor overloads :
 * void operator()(VectorType const & X, VectorType & variance, umintl::gradient_variance_tag tag)
 *
 * The parameter tag contains the information on the current offset and sample size
 */
template<class BackendType>
struct dynamically_sampled : public model_base<BackendType> {
  private:
    typedef typename BackendType::ScalarType ScalarType;
    typedef typename BackendType::VectorType VectorType;

  public:
    dynamically_sampled(double r, size_t fbatch, size_t dataset_size, double theta = 0.5) : theta_(theta), r_(r), S(std::min(fbatch,dataset_size)), offset_(0), H_offset_(0), N(dataset_size){ }

    bool update(optimization_context<BackendType> & c){
      if(S==N){
        H_offset_=(H_offset_+(int)(r_*S))%(S - (int)(r_*S) + 1);
        return false;
      }
      else{
        VectorType var = BackendType::create_vector(c.N());
        c.fun().compute_gradient_variance(c.x(),var,gradient_variance(STOCHASTIC,S,offset_));

        //is_descent_direction = norm1(var)/S*[(N-S)/(N-1)] <= theta^2*norm2(grad)^2
        ScalarType nrm1var = BackendType::asum(c.N(),var);
        ScalarType nrm2grad = BackendType::nrm2(c.N(),c.g());
        //std::gradient_variance << nrm1var*scal << " " << std::pow(theta_,2)*std::pow(nrm2grad,2) << std::endl;
        bool is_descent_direction = (nrm1var/S <= (std::pow(theta_,2)*std::pow(nrm2grad,2)));

        //Update parameters
        //size_t old_S = S;
        if(is_descent_direction==false){
          S = (size_t)(nrm1var/std::pow(theta_*nrm2grad,2));
          S = std::min(S,N);
          if(S>N/2)
            S=N;
        }
        offset_=(offset_+S)%(N-S+1);

        if(is_descent_direction==false)
          H_offset_ = 0;
        else
          H_offset_=(H_offset_+S)%(S - (int)(r_*S) + 1);

        BackendType::delete_if_dynamically_allocated(var);
        return true;
      }
    }

    value_gradient get_value_gradient_tag() const {
      return value_gradient(STOCHASTIC,S,offset_);
    }

    hessian_vector_product get_hv_product_tag() const {
      return hessian_vector_product(STOCHASTIC,(size_t)(r_*S),H_offset_+offset_);
    }
private:
    double theta_;
    double r_;
    size_t S;
    size_t offset_;
    size_t H_offset_;
    size_t N;
};

}
#endif
<|endoftext|>


<commit_before>#ifndef UNESCAPE_COPY_ROW_HPP
#define UNESCAPE_COPY_ROW_HPP

#include <boost/noncopyable.hpp>
#include <boost/fusion/include/size.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/optional.hpp>
#include <boost/fusion/include/for_each.hpp>
#include <boost/format.hpp>

#include "types.hpp"

template <typename S, typename T>
struct unescape_copy_row
  : public boost::noncopyable {
  static const size_t s_num_columns = boost::fusion::result_of::size<T>::value;

  explicit unescape_copy_row(S &source)
  : m_source(source),
    m_reorder(calculate_reorder(m_source.column_names())) {
  }

  ~unescape_copy_row() {
  }

  size_t read(T &row) {
    std::string line;
    size_t num = m_source.read(line);
    if (num > 0) {
      unpack(line, row);
    }
    return num;
  }

private:
  void unpack(std::string &line, T &row) {
    const size_t sz = s_num_columns;
    std::vector<std::pair<char *, size_t> > columns, old_columns;
    {
      char *prev_ptr = &line[0];
      char * const end_ptr = &line[line.size()];
      char *ptr = &line[0];
      for (; ptr != end_ptr; ++ptr) {
        if (*ptr == '\t') {
          *ptr = '\0';
          old_columns.push_back(std::make_pair(prev_ptr, std::distance(prev_ptr, ptr)));
          prev_ptr = ptr + 1;
        }
      }
      old_columns.push_back(std::make_pair(prev_ptr, std::distance(prev_ptr, ptr)));
    }

    columns.reserve(sz);
    for (size_t i = 0; i < sz; ++i) {
      if (i >= m_reorder.size()) {
        BOOST_THROW_EXCEPTION(std::runtime_error((boost::format("Index %1% exceeds m_reorder.size() %2%, this is a bug.")
                                                  % i % m_reorder.size()).str()));
      }
      size_t j = m_reorder[i];
      if (j >= old_columns.size()) {
        BOOST_THROW_EXCEPTION(std::runtime_error("Reordered index exceeds old_columns.size(), this is a bug."));
      }
      columns.push_back(old_columns[j]);
    }

    if (columns.size() != sz) {
      BOOST_THROW_EXCEPTION(std::runtime_error((boost::format("Wrong number of columns: expecting %1%, got %2% in line `%3%'.")
                                                % sz % columns.size() % line).str()));
    }
    try {
      set_values(row, columns);
    } catch (const std::exception &e) {
      BOOST_THROW_EXCEPTION(std::runtime_error((boost::format("%1%: in line `%2%'.") % e.what() % line).str()));
    }
  }

  inline void set_values(T &t, std::vector<std::pair<char *, size_t> > &vs) {
    boost::fusion::for_each(t, set_value(vs.begin()));
  }

  struct set_value {
    explicit set_value(std::vector<std::pair<char *, size_t> >::iterator i) : itr(i) {}

    void operator()(bool &b) const {
      std::pair<char *, size_t> str = *itr++;
      switch (str.first[0]) {
      case 't':
        b = true;
        break;
      case 'f':
        b = false;
        break;
      default:
        BOOST_THROW_EXCEPTION(std::runtime_error((boost::format("Unrecognised value for bool: `%1%'") % str.first).str()));
      }
    }

    void operator()(int16_t &i) const {
      std::pair<char *, size_t> str = *itr++;
      unescape(str);
      i = int16_t(strtol(str.first, NULL, 10));
    }

    void operator()(int32_t &i) const {
      std::pair<char *, size_t> str = *itr++;
      unescape(str);
      i = int32_t(strtol(str.first, NULL, 10));
    }

    void operator()(int64_t &i) const {
      std::pair<char *, size_t> str = *itr++;
      unescape(str);
      i = int64_t(strtoll(str.first, NULL, 10));
    }

    void operator()(double &d) const {
      std::pair<char *, size_t> str = *itr++;
      unescape(str);
      d = strtod(str.first, NULL);
    }

    void operator()(std::string &v) const {
      std::pair<char *, size_t> str = *itr++;
      v.assign(str.first, str.second);
    }

    void operator()(boost::posix_time::ptime &t) const {
      std::pair<char *, size_t> str = *itr++;
      unescape(str);
      //                    11111111112
      //           12345678901234567890
      // format is 2013-09-11 13:39:52.742365
      if (str.second < 19) {
        BOOST_THROW_EXCEPTION(std::runtime_error((boost::format("Unexpected format for timestamp: `%1%'.")
                                                  % str.first).str()));
      }
      int year  = ((str.first[0] - '0') * 1000 +
                   (str.first[1] - '0') * 100 +
                   (str.first[2] - '0') * 10 +
                   (str.first[3] - '0'));
      int month = ((str.first[5] - '0') * 10 + (str.first[6] - '0'));
      int day   = ((str.first[8] - '0') * 10 + (str.first[9] - '0'));
      int hour  = ((str.first[11] - '0') * 10 + (str.first[12] - '0'));
      int min   = ((str.first[14] - '0') * 10 + (str.first[15] - '0'));
      int sec   = ((str.first[17] - '0') * 10 + (str.first[18] - '0'));
      t = boost::posix_time::ptime(boost::gregorian::date(year, month, day),
                                   boost::posix_time::time_duration(hour, min, sec));
    }

    template <typename V>
    void operator()(boost::optional<V> &o) const {
      std::pair<char *, size_t> s = *itr;
      if (strncmp(s.first, "\\N", s.second) == 0) {
        o = boost::none;
        ++itr;
      } else {
        V v;
        operator()(v);
        o = v;
      }
    }

    void operator()(user_status_enum &e) const {
      std::pair<char *, size_t> str = *itr++;
      unescape(str);
      if (strncmp(str.first, "pending", str.second) == 0) {
        e = user_status_pending;
      } else if (strncmp(str.first, "active", str.second) == 0) {
        e = user_status_active;
      } else if (strncmp(str.first, "confirmed", str.second) == 0) {
        e = user_status_confirmed;
      } else if (strncmp(str.first, "suspended", str.second) == 0) {
        e = user_status_suspended;
      } else if (strncmp(str.first, "deleted", str.second) == 0) {
        e = user_status_deleted;
      } else {
        BOOST_THROW_EXCEPTION(std::runtime_error((boost::format("Unrecognised value for user_status_enum: `%1%'.") % str.first).str()));
      }
    }

    void operator()(format_enum &e) const {
      std::pair<char *, size_t> str = *itr++;
      unescape(str);
      if (strncmp(str.first, "html", str.second) == 0) {
        e = format_html;
      } else if (strncmp(str.first, "markdown", str.second) == 0) {
        e = format_markdown;
      } else if (strncmp(str.first, "text", str.second) == 0) {
        e = format_text;
      } else {
        BOOST_THROW_EXCEPTION(std::runtime_error((boost::format("Unrecognised value for format_enum: `%1%'.") % str.first).str()));
      }
    }

    void operator()(nwr_enum &e) const {
      std::pair<char *, size_t> str = *itr++;
      unescape(str);
      if (strncmp(str.first, "Node", str.second) == 0) {
        e = nwr_node;
      } else if (strncmp(str.first, "Way", str.second) == 0) {
        e = nwr_way;
      } else if (strncmp(str.first, "Relation", str.second) == 0) {
        e = nwr_relation;
      } else {
        BOOST_THROW_EXCEPTION(std::runtime_error((boost::format("Unrecognised value for nwr_enum: `%1%'.") % str.first).str()));
      }
    }

    inline int hex2digit(char ch) const {
      switch (ch) {
      case '0':
      case '1':
      case '2':
      case '3':
      case '4':
      case '5':
      case '6':
      case '7':
      case '8':
      case '9':
        return int(ch - '0');

      case 'a':
      case 'b':
      case 'c':
      case 'd':
      case 'e':
      case 'f':
        return 10 + int(ch - 'a');

      case 'A':
      case 'B':
      case 'C':
      case 'D':
      case 'E':
      case 'F':
        return 10 + int(ch - 'A');

      default:
        BOOST_THROW_EXCEPTION(std::runtime_error("Invalid hex digit."));
      }
    }

    inline int oct2digit(char ch) const {
      if ((ch >= '0') && (ch <= '7')) {
        return int(ch - '0');
      } else {
        BOOST_THROW_EXCEPTION(std::runtime_error("Invalid octal digit."));
      }
    }

    void unescape(std::pair<char *, size_t> &s) const {
      const size_t end = s.second;
      char *str = s.first;
      size_t j = 0;

      for (size_t i = 0; i < end; ++i) {
        switch (str[i]) {
        case '\\':
          ++i;
          if (i < end) {
            switch (str[i]) {
            case 'b':
              str[j] = '\b';
              break;

            case 'f':
              str[j] = '\f';
              break;

            case 'n':
              str[j] = '\n';
              break;

            case 'r':
              str[j] = '\r';
              break;

            case 't':
              str[j] = '\t';
              break;

            case 'v':
              str[j] = '\v';
              break;

            case 'x':
              i += 2;
              if (i < end) {
              } else {
                str[j] = char(hex2digit(str[i-1]) * 16 + hex2digit(str[i]));
                BOOST_THROW_EXCEPTION(std::runtime_error("Unterminated hex escape sequence."));
              }
              break;

            case '0':
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
              i += 2;
              if (i < end) {
                str[j] = char(oct2digit(str[i-2]) * 64 + oct2digit(str[i-1]) * 8 + oct2digit(str[i]));
              } else {
                BOOST_THROW_EXCEPTION(std::runtime_error("Unterminated octal escape sequence."));
              }
              break;

            default:
              // an unnecessary escape
              str[j] = str[i];
            }

          } else {
            BOOST_THROW_EXCEPTION(std::runtime_error("Unterminated escape sequence."));
          }
          break;

        default:
          if (i != j) {
            str[j] = str[i];
          }
        }

        ++j;
      }

      str[j] = '\0';
      s.second = j;
    }

    mutable std::vector<std::pair<char *, size_t> >::iterator itr;
  };

  static std::vector<size_t> calculate_reorder(const std::vector<std::string> &names) {
    std::vector<size_t> indexes;
    const std::vector<std::string> &wanted_names = T::column_names();

    const size_t num_columns = wanted_names.size();
    indexes.reserve(num_columns);
    for (size_t i = 0; i < num_columns; ++i) {
      const std::string &wanted_name = wanted_names[i];
      size_t j = i;

      if (wanted_name != "*") {
        std::vector<std::string>::const_iterator itr = std::find(names.begin(), names.end(), wanted_name);
        if (itr == names.end()) {
          std::ostringstream ostr;
          ostr << "Unable to find wanted column name \"" << wanted_name << "\" in available names: ";
          for (std::vector<std::string>::const_iterator jtr = names.begin(); jtr != names.end(); ++jtr) {
            ostr << "\"" << *jtr << "\", ";
          }
          BOOST_THROW_EXCEPTION(std::runtime_error(ostr.str()));
        }
        j = std::distance(names.begin(), itr);
      }

      indexes.push_back(j);
    }

    return indexes;
  }

  S &m_source;
  std::vector<size_t> m_reorder;
};

#endif /* UNESCAPE_COPY_ROW_HPP */
<commit_msg>Remember to unescape strings from the PostgreSQL dump as well.<commit_after>#ifndef UNESCAPE_COPY_ROW_HPP
#define UNESCAPE_COPY_ROW_HPP

#include <boost/noncopyable.hpp>
#include <boost/fusion/include/size.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/optional.hpp>
#include <boost/fusion/include/for_each.hpp>
#include <boost/format.hpp>

#include "types.hpp"

template <typename S, typename T>
struct unescape_copy_row
  : public boost::noncopyable {
  static const size_t s_num_columns = boost::fusion::result_of::size<T>::value;

  explicit unescape_copy_row(S &source)
  : m_source(source),
    m_reorder(calculate_reorder(m_source.column_names())) {
  }

  ~unescape_copy_row() {
  }

  size_t read(T &row) {
    std::string line;
    size_t num = m_source.read(line);
    if (num > 0) {
      unpack(line, row);
    }
    return num;
  }

private:
  void unpack(std::string &line, T &row) {
    const size_t sz = s_num_columns;
    std::vector<std::pair<char *, size_t> > columns, old_columns;
    {
      char *prev_ptr = &line[0];
      char * const end_ptr = &line[line.size()];
      char *ptr = &line[0];
      for (; ptr != end_ptr; ++ptr) {
        if (*ptr == '\t') {
          *ptr = '\0';
          old_columns.push_back(std::make_pair(prev_ptr, std::distance(prev_ptr, ptr)));
          prev_ptr = ptr + 1;
        }
      }
      old_columns.push_back(std::make_pair(prev_ptr, std::distance(prev_ptr, ptr)));
    }

    columns.reserve(sz);
    for (size_t i = 0; i < sz; ++i) {
      if (i >= m_reorder.size()) {
        BOOST_THROW_EXCEPTION(std::runtime_error((boost::format("Index %1% exceeds m_reorder.size() %2%, this is a bug.")
                                                  % i % m_reorder.size()).str()));
      }
      size_t j = m_reorder[i];
      if (j >= old_columns.size()) {
        BOOST_THROW_EXCEPTION(std::runtime_error("Reordered index exceeds old_columns.size(), this is a bug."));
      }
      columns.push_back(old_columns[j]);
    }

    if (columns.size() != sz) {
      BOOST_THROW_EXCEPTION(std::runtime_error((boost::format("Wrong number of columns: expecting %1%, got %2% in line `%3%'.")
                                                % sz % columns.size() % line).str()));
    }
    try {
      set_values(row, columns);
    } catch (const std::exception &e) {
      BOOST_THROW_EXCEPTION(std::runtime_error((boost::format("%1%: in line `%2%'.") % e.what() % line).str()));
    }
  }

  inline void set_values(T &t, std::vector<std::pair<char *, size_t> > &vs) {
    boost::fusion::for_each(t, set_value(vs.begin()));
  }

  struct set_value {
    explicit set_value(std::vector<std::pair<char *, size_t> >::iterator i) : itr(i) {}

    void operator()(bool &b) const {
      std::pair<char *, size_t> str = *itr++;
      switch (str.first[0]) {
      case 't':
        b = true;
        break;
      case 'f':
        b = false;
        break;
      default:
        BOOST_THROW_EXCEPTION(std::runtime_error((boost::format("Unrecognised value for bool: `%1%'") % str.first).str()));
      }
    }

    void operator()(int16_t &i) const {
      std::pair<char *, size_t> str = *itr++;
      unescape(str);
      i = int16_t(strtol(str.first, NULL, 10));
    }

    void operator()(int32_t &i) const {
      std::pair<char *, size_t> str = *itr++;
      unescape(str);
      i = int32_t(strtol(str.first, NULL, 10));
    }

    void operator()(int64_t &i) const {
      std::pair<char *, size_t> str = *itr++;
      unescape(str);
      i = int64_t(strtoll(str.first, NULL, 10));
    }

    void operator()(double &d) const {
      std::pair<char *, size_t> str = *itr++;
      unescape(str);
      d = strtod(str.first, NULL);
    }

    void operator()(std::string &v) const {
      std::pair<char *, size_t> str = *itr++;
      unescape(str);
      v.assign(str.first, str.second);
    }

    void operator()(boost::posix_time::ptime &t) const {
      std::pair<char *, size_t> str = *itr++;
      unescape(str);
      //                    11111111112
      //           12345678901234567890
      // format is 2013-09-11 13:39:52.742365
      if (str.second < 19) {
        BOOST_THROW_EXCEPTION(std::runtime_error((boost::format("Unexpected format for timestamp: `%1%'.")
                                                  % str.first).str()));
      }
      int year  = ((str.first[0] - '0') * 1000 +
                   (str.first[1] - '0') * 100 +
                   (str.first[2] - '0') * 10 +
                   (str.first[3] - '0'));
      int month = ((str.first[5] - '0') * 10 + (str.first[6] - '0'));
      int day   = ((str.first[8] - '0') * 10 + (str.first[9] - '0'));
      int hour  = ((str.first[11] - '0') * 10 + (str.first[12] - '0'));
      int min   = ((str.first[14] - '0') * 10 + (str.first[15] - '0'));
      int sec   = ((str.first[17] - '0') * 10 + (str.first[18] - '0'));
      t = boost::posix_time::ptime(boost::gregorian::date(year, month, day),
                                   boost::posix_time::time_duration(hour, min, sec));
    }

    template <typename V>
    void operator()(boost::optional<V> &o) const {
      std::pair<char *, size_t> s = *itr;
      if (strncmp(s.first, "\\N", s.second) == 0) {
        o = boost::none;
        ++itr;
      } else {
        V v;
        operator()(v);
        o = v;
      }
    }

    void operator()(user_status_enum &e) const {
      std::pair<char *, size_t> str = *itr++;
      unescape(str);
      if (strncmp(str.first, "pending", str.second) == 0) {
        e = user_status_pending;
      } else if (strncmp(str.first, "active", str.second) == 0) {
        e = user_status_active;
      } else if (strncmp(str.first, "confirmed", str.second) == 0) {
        e = user_status_confirmed;
      } else if (strncmp(str.first, "suspended", str.second) == 0) {
        e = user_status_suspended;
      } else if (strncmp(str.first, "deleted", str.second) == 0) {
        e = user_status_deleted;
      } else {
        BOOST_THROW_EXCEPTION(std::runtime_error((boost::format("Unrecognised value for user_status_enum: `%1%'.") % str.first).str()));
      }
    }

    void operator()(format_enum &e) const {
      std::pair<char *, size_t> str = *itr++;
      unescape(str);
      if (strncmp(str.first, "html", str.second) == 0) {
        e = format_html;
      } else if (strncmp(str.first, "markdown", str.second) == 0) {
        e = format_markdown;
      } else if (strncmp(str.first, "text", str.second) == 0) {
        e = format_text;
      } else {
        BOOST_THROW_EXCEPTION(std::runtime_error((boost::format("Unrecognised value for format_enum: `%1%'.") % str.first).str()));
      }
    }

    void operator()(nwr_enum &e) const {
      std::pair<char *, size_t> str = *itr++;
      unescape(str);
      if (strncmp(str.first, "Node", str.second) == 0) {
        e = nwr_node;
      } else if (strncmp(str.first, "Way", str.second) == 0) {
        e = nwr_way;
      } else if (strncmp(str.first, "Relation", str.second) == 0) {
        e = nwr_relation;
      } else {
        BOOST_THROW_EXCEPTION(std::runtime_error((boost::format("Unrecognised value for nwr_enum: `%1%'.") % str.first).str()));
      }
    }

    inline int hex2digit(char ch) const {
      switch (ch) {
      case '0':
      case '1':
      case '2':
      case '3':
      case '4':
      case '5':
      case '6':
      case '7':
      case '8':
      case '9':
        return int(ch - '0');

      case 'a':
      case 'b':
      case 'c':
      case 'd':
      case 'e':
      case 'f':
        return 10 + int(ch - 'a');

      case 'A':
      case 'B':
      case 'C':
      case 'D':
      case 'E':
      case 'F':
        return 10 + int(ch - 'A');

      default:
        BOOST_THROW_EXCEPTION(std::runtime_error("Invalid hex digit."));
      }
    }

    inline int oct2digit(char ch) const {
      if ((ch >= '0') && (ch <= '7')) {
        return int(ch - '0');
      } else {
        BOOST_THROW_EXCEPTION(std::runtime_error("Invalid octal digit."));
      }
    }

    void unescape(std::pair<char *, size_t> &s) const {
      const size_t end = s.second;
      char *str = s.first;
      size_t j = 0;

      for (size_t i = 0; i < end; ++i) {
        switch (str[i]) {
        case '\\':
          ++i;
          if (i < end) {
            switch (str[i]) {
            case 'b':
              str[j] = '\b';
              break;

            case 'f':
              str[j] = '\f';
              break;

            case 'n':
              str[j] = '\n';
              break;

            case 'r':
              str[j] = '\r';
              break;

            case 't':
              str[j] = '\t';
              break;

            case 'v':
              str[j] = '\v';
              break;

            case 'x':
              i += 2;
              if (i < end) {
              } else {
                str[j] = char(hex2digit(str[i-1]) * 16 + hex2digit(str[i]));
                BOOST_THROW_EXCEPTION(std::runtime_error("Unterminated hex escape sequence."));
              }
              break;

            case '0':
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
              i += 2;
              if (i < end) {
                str[j] = char(oct2digit(str[i-2]) * 64 + oct2digit(str[i-1]) * 8 + oct2digit(str[i]));
              } else {
                BOOST_THROW_EXCEPTION(std::runtime_error("Unterminated octal escape sequence."));
              }
              break;

            default:
              // an unnecessary escape
              str[j] = str[i];
            }

          } else {
            BOOST_THROW_EXCEPTION(std::runtime_error("Unterminated escape sequence."));
          }
          break;

        default:
          if (i != j) {
            str[j] = str[i];
          }
        }

        ++j;
      }

      str[j] = '\0';
      s.second = j;
    }

    mutable std::vector<std::pair<char *, size_t> >::iterator itr;
  };

  static std::vector<size_t> calculate_reorder(const std::vector<std::string> &names) {
    std::vector<size_t> indexes;
    const std::vector<std::string> &wanted_names = T::column_names();

    const size_t num_columns = wanted_names.size();
    indexes.reserve(num_columns);
    for (size_t i = 0; i < num_columns; ++i) {
      const std::string &wanted_name = wanted_names[i];
      size_t j = i;

      if (wanted_name != "*") {
        std::vector<std::string>::const_iterator itr = std::find(names.begin(), names.end(), wanted_name);
        if (itr == names.end()) {
          std::ostringstream ostr;
          ostr << "Unable to find wanted column name \"" << wanted_name << "\" in available names: ";
          for (std::vector<std::string>::const_iterator jtr = names.begin(); jtr != names.end(); ++jtr) {
            ostr << "\"" << *jtr << "\", ";
          }
          BOOST_THROW_EXCEPTION(std::runtime_error(ostr.str()));
        }
        j = std::distance(names.begin(), itr);
      }

      indexes.push_back(j);
    }

    return indexes;
  }

  S &m_source;
  std::vector<size_t> m_reorder;
};

#endif /* UNESCAPE_COPY_ROW_HPP */
<|endoftext|>