<commit_before>// Copyright 2018, Intel Corporation

#include "tile/codegen/autotile.h"

#include "base/util/logging.h"
#include "base/util/stream_container.h"
#include "base/util/throw.h"
#include "tile/codegen/math.h"
#include "tile/codegen/tags.h"
#include "tile/codegen/tile.h"
#include "tile/stripe/stripe.h"

namespace vertexai {
namespace tile {
namespace codegen {

using namespace stripe;  // NOLINT

namespace {

struct TileSizes {
  int64_t inputs = 0;
  int64_t outputs = 0;
  int64_t total = 0;
};

struct TileDimension {
  size_t range = 0;
  size_t size = 0;
};

struct Tile {
  explicit Tile(const Block& block) : dims(block.idxs.size()) {
    for (size_t i = 0; i < block.idxs.size(); i++) {
      set(i, 1, block.idxs[i].range);
    }
  }

  std::vector<TileDimension> dims;

  void set(size_t i, size_t value, size_t range) {
    value = std::min(value, range);
    dims[i].range = value;
    dims[i].size = IntDivCeil(range, value);
  }

  size_t product() const {
    size_t ret = 1;
    for (const auto& dim : dims) {
      ret *= dim.range;
    }
    return ret;
  }

  TileShape ranges() const {
    TileShape ret(dims.size());
    for (size_t i = 0; i < dims.size(); i++) {
      ret[i] = dims[i].range;
    }
    return ret;
  }

  TileShape sizes() const {
    TileShape ret(dims.size());
    for (size_t i = 0; i < dims.size(); i++) {
      ret[i] = dims[i].size;
    }
    return ret;
  }
};

bool operator<(const TileDimension& lhs, const TileDimension& rhs) {
  return std::tie(lhs.range, lhs.size) < std::tie(rhs.range, rhs.size);
}

bool operator<(const Tile& lhs, const Tile& rhs) {  //
  return lhs.dims < rhs.dims;
}

std::ostream& operator<<(std::ostream& os, const TileSizes& sizes) {
  os << "(" << sizes.inputs << ", " << sizes.outputs << ", " << sizes.total << ")";
  return os;
}

std::ostream& operator<<(std::ostream& os, const TileDimension& dim) {
  os << dim.range << ":" << dim.size;
  return os;
}

std::ostream& operator<<(std::ostream& os, const Tile& tile) {
  os << StreamContainer(tile.dims);
  return os;
}

size_t ComputeSize(const std::map<std::string, size_t>& tile_by_name,  //
                   const Refinement& ref) {
  size_t total_size = 1;
  Affine flat = ref.FlatAccess();
  for (const auto& kvp : flat.getMap()) {
    if (!kvp.first.empty()) {
      total_size *= tile_by_name.at(kvp.first);
    }
  }
  IVLOG(4, "    ComputeSize> ref: " << ref.into << ", total_size: " << total_size);
  return total_size;
}

size_t ComputeBytes(const std::map<std::string, size_t>& tile_by_name,  //
                    const Refinement& ref) {
  auto tiled = ref;  // make a mutable copy
  tiled.ApplyTile(tile_by_name);
  auto bytes = tiled.shape.byte_size();
  IVLOG(4, "    ComputeBytes> ref: " << ref);
  IVLOG(4, "                tiled: " << tiled);
  IVLOG(4, "                bytes: " << bytes);
  return bytes;
}

TileSizes ComputeSizes(const std::map<std::string, size_t>& tile_by_name,  //
                       const Block& block,                                 //
                       const proto::AutotilePass& options) {
  TileSizes ret;
  for (const auto& ref : block.refs) {
    if (ref.dir == RefDir::None) {
      continue;
    }
    if (options.skip_1d() && ref.shape.dims.size() == 1) {
      continue;
    }
    auto size = options.use_bytes() ? ComputeBytes(tile_by_name, ref)  //
                                    : ComputeSize(tile_by_name, ref);
    ret.total += size;
    if (ref.dir == RefDir::In) {
      ret.inputs += size;
    } else if (ref.dir == RefDir::Out) {
      ret.outputs += size;
    }
  }
  return ret;
}

double TileCost(const Block& block, const proto::AutotilePass& options, const Tile& tile) {
  std::map<std::string, size_t> tile_by_name;
  for (size_t i = 0; i < block.idxs.size(); i++) {
    tile_by_name[block.idxs[i].name] = options.use_bytes() ? tile.dims[i].size : tile.dims[i].range;
  }
  auto sizes = ComputeSizes(tile_by_name, block, options);
  IVLOG(3, "    TileCost> tile_by_name: " << tile_by_name << ", sizes: " << sizes);
  if ((options.max_output_size() && sizes.outputs > options.max_output_size()) ||
      (options.max_input_size() && sizes.inputs > options.max_input_size()) ||
      (options.max_total_size() && sizes.total > options.max_total_size())) {
    return std::numeric_limits<double>::infinity();
  }
  double total_compute = 1;
  double tile_expand = 1;
  for (size_t i = 0; i < block.idxs.size(); i++) {
    const auto& tile_dim = tile.dims[i];
    total_compute *= tile_by_name[block.idxs[i].name];
    size_t padded_size = tile_dim.range * tile_dim.size;
    tile_expand *= static_cast<double>(padded_size) / static_cast<double>(block.idxs[i].range);
  }
  auto input_cost = options.input_cost() * sizes.inputs;
  auto output_cost = options.output_cost() * sizes.outputs;
  double cost = tile_expand * (output_cost + input_cost) / total_compute;
  IVLOG(3, "        cost: " << cost);
  return cost;
}

Tile PickBestTile(const Block& block, const proto::AutotilePass& options) {
  IVLOG(2, "Autotile> block: " << block.name);
  std::multimap<double, Tile> by_cost;
  std::map<Tile, double> by_tile;
  std::set<std::pair<double, Tile>> to_do;
  Tile tile(block);
  double cost = TileCost(block, options, tile);
  double base_cost = cost;
  by_tile.emplace(tile, cost);
  by_cost.emplace(cost, tile);
  to_do.emplace(cost, tile);
  while (!to_do.empty()) {
    auto it = to_do.begin();
    if (it->first > cost && options.fast()) {
      break;
    }
    cost = it->first;
    tile = it->second;
    to_do.erase(*it);
    for (size_t i = 0; i < block.idxs.size(); i++) {
      if (block.idxs[i].has_tag("bank")) {
        continue;
      }
      auto prev = tile.dims[i];
      if (options.only_po2()) {
        tile.set(i, 2 * tile.dims[i].range, block.idxs[i].range);
      } else {
        tile.set(i, tile.dims[i].range + 1, block.idxs[i].range);
      }
      if (!by_tile.count(tile)) {
        cost = TileCost(block, options, tile);
        by_tile.emplace(tile, cost);
        by_cost.emplace(cost, tile);
        if (!std::isinf(cost)) {
          to_do.emplace(cost, tile);
        }
      }
      tile.dims[i] = prev;
    }
  }
  auto winner = by_cost.begin();
  IVLOG(4, "Tiles: ");
  for (const auto& kvp : by_cost) {
    IVLOG(4, "    " << kvp.first << ": " << kvp.second);
  }
  IVLOG(2, "    best: " << winner->second << ", cost: " << winner->first / base_cost);
  return winner->second;
}

bool TileViaStrategy(Block* block, const Tile& tile, const proto::AutotilePass& options) {
  IVLOG(2, "    TileViaStrategy: " << tile);
  switch (options.strategy()) {
    case proto::TileStrategy::TileRanges:
      return ApplyTile(block, tile.ranges());
    case proto::TileStrategy::TileSizes:
      // return ExtractTile(block, tile, options.into_idx());
      return ApplyTile(block, tile.sizes());
    case proto::TileStrategy::TileSplitIndex:
      // TODO
      break;
  }
  throw_with_trace(std::runtime_error("Unsupported tiling strategy"));
}

}  // namespace

void AutotilePass(Block* root, const proto::AutotilePass& options) {
  auto reqs = FromProto(options.reqs());
  RunOnBlocks(root, reqs, [&options](const AliasMap& map, Block* block) {
    auto tile = PickBestTile(*block, options);
    if (tile.product() > 1 && TileViaStrategy(block, tile, options)) {
      auto inner = block->SubBlock(0);
      if (options.copy_tags()) {
        inner->tags = block->tags;
      }
      if (options.drop_outer_tags()) {
        block->tags.clear();
      }
      block->add_tags(FromProto(options.outer_set()));
      inner->add_tags(FromProto(options.inner_set()));
      if (!options.idx_tag().empty()) {
        for (auto& idx : block->idxs) {
          if (idx.range > 1) {
            idx.set_tag(options.idx_tag());
          }
          // HACK: remove this somehow
          idx.tags.erase("bank");
        }
      }
    }
  });
}

}  // namespace codegen
}  // namespace tile
}  // namespace vertexai
<commit_msg>Autotile should keep trying if nothing good has been found<commit_after>// Copyright 2018, Intel Corporation

#include "tile/codegen/autotile.h"

#include "base/util/logging.h"
#include "base/util/stream_container.h"
#include "base/util/throw.h"
#include "tile/codegen/math.h"
#include "tile/codegen/tags.h"
#include "tile/codegen/tile.h"
#include "tile/stripe/stripe.h"

namespace vertexai {
namespace tile {
namespace codegen {

using namespace stripe;  // NOLINT

namespace {

struct TileMetrics {
  int64_t inputs = 0;
  int64_t outputs = 0;
  int64_t total = 0;
};

struct TileDimension {
  size_t range = 0;
  size_t size = 0;
};

struct Tile {
  explicit Tile(const Block& block) : dims(block.idxs.size()) {
    for (size_t i = 0; i < block.idxs.size(); i++) {
      set(i, 1, block.idxs[i].range);
    }
  }

  std::vector<TileDimension> dims;
  TileMetrics metrics;

  void set(size_t i, size_t value, size_t range) {
    value = std::min(value, range);
    dims[i].range = value;
    dims[i].size = IntDivCeil(range, value);
  }

  size_t product() const {
    size_t ret = 1;
    for (const auto& dim : dims) {
      ret *= dim.range;
    }
    return ret;
  }

  TileShape ranges() const {
    TileShape ret(dims.size());
    for (size_t i = 0; i < dims.size(); i++) {
      ret[i] = dims[i].range;
    }
    return ret;
  }

  TileShape sizes() const {
    TileShape ret(dims.size());
    for (size_t i = 0; i < dims.size(); i++) {
      ret[i] = dims[i].size;
    }
    return ret;
  }

  bool IsValid(const proto::AutotilePass& options) const {
    return !((options.max_output_size() && metrics.outputs > options.max_output_size()) ||
             (options.max_input_size() && metrics.inputs > options.max_input_size()) ||
             (options.max_total_size() && metrics.total > options.max_total_size()));
  }
};

bool operator<(const TileDimension& lhs, const TileDimension& rhs) {
  return std::tie(lhs.range, lhs.size) < std::tie(rhs.range, rhs.size);
}

bool operator<(const Tile& lhs, const Tile& rhs) {  //
  return lhs.dims < rhs.dims;
}

std::ostream& operator<<(std::ostream& os, const TileMetrics& metrics) {
  os << "(" << metrics.inputs << ", " << metrics.outputs << ", " << metrics.total << ")";
  return os;
}

std::ostream& operator<<(std::ostream& os, const TileDimension& dim) {
  os << dim.range << ":" << dim.size;
  return os;
}

std::ostream& operator<<(std::ostream& os, const Tile& tile) {
  os << StreamContainer(tile.dims);
  return os;
}

size_t ComputeSize(const std::map<std::string, size_t>& tile_by_name,  //
                   const Refinement& ref) {
  size_t total_size = 1;
  Affine flat = ref.FlatAccess();
  for (const auto& kvp : flat.getMap()) {
    if (!kvp.first.empty()) {
      total_size *= tile_by_name.at(kvp.first);
    }
  }
  IVLOG(4, "    ComputeSize> ref: " << ref.into << ", total_size: " << total_size);
  return total_size;
}

size_t ComputeBytes(const std::map<std::string, size_t>& tile_by_name,  //
                    const Refinement& ref) {
  auto tiled = ref;  // make a mutable copy
  tiled.ApplyTile(tile_by_name);
  auto bytes = tiled.shape.byte_size();
  IVLOG(4, "    ComputeBytes> ref: " << ref);
  IVLOG(4, "                tiled: " << tiled);
  IVLOG(4, "                bytes: " << bytes);
  return bytes;
}

TileMetrics ComputeSizes(const std::map<std::string, size_t>& tile_by_name,  //
                         const Block& block,                                 //
                         const proto::AutotilePass& options) {
  TileMetrics ret;
  for (const auto& ref : block.refs) {
    if (ref.dir == RefDir::None) {
      continue;
    }
    if (options.skip_1d() && ref.shape.dims.size() == 1) {
      continue;
    }
    auto size = options.use_bytes() ? ComputeBytes(tile_by_name, ref)  //
                                    : ComputeSize(tile_by_name, ref);
    ret.total += size;
    if (ref.dir == RefDir::In) {
      ret.inputs += size;
    } else if (ref.dir == RefDir::Out) {
      ret.outputs += size;
    }
  }
  return ret;
}

double TileCost(const Block& block, const proto::AutotilePass& options, Tile* tile) {
  std::map<std::string, size_t> tile_by_name;
  for (size_t i = 0; i < block.idxs.size(); i++) {
    tile_by_name[block.idxs[i].name] = options.use_bytes() ? tile->dims[i].size : tile->dims[i].range;
  }
  tile->metrics = ComputeSizes(tile_by_name, block, options);
  IVLOG(3, "    TileCost> tile_by_name: " << tile_by_name << ", metrics: " << tile->metrics);
  if (!tile->IsValid(options)) {
    return std::numeric_limits<double>::infinity();
  }
  double total_compute = 1;
  double tile_expand = 1;
  for (size_t i = 0; i < block.idxs.size(); i++) {
    const auto& tile_dim = tile->dims[i];
    total_compute *= tile_by_name[block.idxs[i].name];
    size_t padded_size = tile_dim.range * tile_dim.size;
    tile_expand *= static_cast<double>(padded_size) / static_cast<double>(block.idxs[i].range);
  }
  auto input_cost = options.input_cost() * tile->metrics.inputs;
  auto output_cost = options.output_cost() * tile->metrics.outputs;
  double cost = tile_expand * (output_cost + input_cost) / total_compute;
  IVLOG(3, "        cost: " << cost);
  return cost;
}

Tile PickBestTile(const Block& block, const proto::AutotilePass& options) {
  IVLOG(3, "Autotile> PickBestTile> block: " << block.name);
  std::multimap<double, Tile> by_cost;
  std::map<Tile, double> by_tile;
  std::set<std::pair<double, Tile>> to_do;
  Tile tile(block);
  double cost = TileCost(block, options, &tile);
  double base_cost = cost;
  bool any_valid = tile.IsValid(options);
  by_tile.emplace(tile, cost);
  by_cost.emplace(cost, tile);
  to_do.emplace(cost, tile);
  while (!to_do.empty()) {
    auto it = to_do.begin();
    if (it->first > cost && options.fast()) {
      break;
    }
    cost = it->first;
    tile = it->second;
    to_do.erase(*it);
    for (size_t i = 0; i < block.idxs.size(); i++) {
      if (block.idxs[i].has_tag("bank")) {
        continue;
      }
      auto prev = tile.dims[i];
      if (options.only_po2()) {
        tile.set(i, 2 * tile.dims[i].range, block.idxs[i].range);
      } else {
        tile.set(i, tile.dims[i].range + 1, block.idxs[i].range);
      }
      if (!by_tile.count(tile)) {
        cost = TileCost(block, options, &tile);
        any_valid |= tile.IsValid(options);
        by_tile.emplace(tile, cost);
        by_cost.emplace(cost, tile);
        if (!any_valid || !std::isinf(cost)) {
          to_do.emplace(cost, tile);
        }
      }
      tile.dims[i] = prev;
    }
  }
  auto winner = by_cost.begin();
  IVLOG(4, "Tiles: ");
  for (const auto& kvp : by_cost) {
    IVLOG(4, "    " << kvp.first << ": " << kvp.second);
  }
  IVLOG(3, "    best: " << winner->second << ", cost: " << winner->first / base_cost);
  return winner->second;
}

bool TileViaStrategy(Block* block, const Tile& tile, const proto::AutotilePass& options) {
  IVLOG(2, "Autotile> block: " << block->name         //
                               << ", tile: " << tile  //
                               << ", metrics: " << tile.metrics);
  switch (options.strategy()) {
    case proto::TileStrategy::TileRanges:
      return ApplyTile(block, tile.ranges());
    case proto::TileStrategy::TileSizes:
      return ApplyTile(block, tile.sizes());
    case proto::TileStrategy::TileSplitIndex:
      // TODO
      break;
  }
  throw_with_trace(std::runtime_error("Unsupported tiling strategy"));
}

}  // namespace

void AutotilePass(Block* root, const proto::AutotilePass& options) {
  auto reqs = FromProto(options.reqs());
  RunOnBlocks(root, reqs, [&options](const AliasMap& map, Block* block) {
    auto tile = PickBestTile(*block, options);
    if (tile.product() > 1 && TileViaStrategy(block, tile, options)) {
      auto inner = block->SubBlock(0);
      if (options.copy_tags()) {
        inner->tags = block->tags;
      }
      if (options.drop_outer_tags()) {
        block->tags.clear();
      }
      block->add_tags(FromProto(options.outer_set()));
      inner->add_tags(FromProto(options.inner_set()));
      if (!options.idx_tag().empty()) {
        for (auto& idx : block->idxs) {
          if (idx.range > 1) {
            idx.set_tag(options.idx_tag());
          }
          // HACK: remove this somehow
          idx.tags.erase("bank");
        }
      }
    } else if (!tile.IsValid(options)) {
      LOG(WARNING) << "Autotile> block: " << block->name << " was NOT split: " << tile.metrics;
    }
  });
}

}  // namespace codegen
}  // namespace tile
}  // namespace vertexai
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2011 Collabora Ltd. <http://www.collabora.co.uk/>
 *   @author Andre Moreira Magalhaes <andre.magalhaes@collabora.co.uk>
 * Copyright (C) 2013 Dan Vrátil <dvratil@redhat.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */

#include "tls-cert-verifier-op.h"

#include <TelepathyQt/PendingVariantMap>

#include <KMessageBox>
#include <KLocalizedString>
#include <KDebug>

#include <QSslCertificate>
#include <QSslCipher>

#include "kssl/ksslcertificatemanager.h"
#include "kssl/ksslinfodialog.h"

#include <QtCrypto/QtCrypto>

TlsCertVerifierOp::TlsCertVerifierOp(const Tp::AccountPtr &account,
        const Tp::ConnectionPtr &connection,
        const Tp::ChannelPtr &channel)
    : Tp::PendingOperation(channel),
      m_account(account),
      m_connection(connection),
      m_channel(channel)
{
    QDBusObjectPath certificatePath = qdbus_cast<QDBusObjectPath>(channel->immutableProperties().value(
                TP_QT_IFACE_CHANNEL_TYPE_SERVER_TLS_CONNECTION + QLatin1String(".ServerCertificate")));
    m_hostname = qdbus_cast<QString>(channel->immutableProperties().value(
                TP_QT_IFACE_CHANNEL_TYPE_SERVER_TLS_CONNECTION + QLatin1String(".Hostname")));
    m_referenceIdentities = qdbus_cast<QStringList>(channel->immutableProperties().value(
                TP_QT_IFACE_CHANNEL_TYPE_SERVER_TLS_CONNECTION + QLatin1String(".ReferenceIdentities")));

    m_authTLSCertificateIface = new Tp::Client::AuthenticationTLSCertificateInterface(
            channel->dbusConnection(), channel->busName(), certificatePath.path());
    connect(m_authTLSCertificateIface->requestAllProperties(),
            SIGNAL(finished(Tp::PendingOperation*)),
            SLOT(gotProperties(Tp::PendingOperation*)));
}

TlsCertVerifierOp::~TlsCertVerifierOp()
{
}

void TlsCertVerifierOp::gotProperties(Tp::PendingOperation *op)
{
    if (op->isError()) {
        kWarning() << "Unable to retrieve properties from AuthenticationTLSCertificate object at" <<
            m_authTLSCertificateIface->path();
        m_channel->requestClose();
        setFinishedWithError(op->errorName(), op->errorMessage());
        return;
    }

    // everything ok, we can return from handleChannels now
    Q_EMIT ready(this);

    Tp::PendingVariantMap *pvm = qobject_cast<Tp::PendingVariantMap*>(op);
    QVariantMap props = qdbus_cast<QVariantMap>(pvm->result());
    m_certType = qdbus_cast<QString>(props.value(QLatin1String("CertificateType")));
    m_certData = qdbus_cast<CertificateDataList>(props.value(QLatin1String("CertificateChainData")));

    if(m_certType.compare(QLatin1String("x509"), Qt::CaseInsensitive)) {
        Tp::TLSCertificateRejectionList rejections;
        m_authTLSCertificateIface->Reject(rejections);
        m_channel->requestClose();
        setFinishedWithError(QLatin1String("Cert.Unknown"),
                             QLatin1String("Invalid certificate type"));
        return;
    }

    // Initialize QCA module
    QCA::Initializer initializer;

    QCA::CertificateChain chain;
    Q_FOREACH (const QByteArray &data, m_certData) {
        chain << QCA::Certificate::fromDER(data);
    }

    if (verifyCertChain(chain)) {
        m_authTLSCertificateIface->Accept().waitForFinished();
        setFinished();
    } else {
        Tp::TLSCertificateRejectionList rejections;
        m_authTLSCertificateIface->Reject(rejections);
        m_channel->requestClose();
        setFinishedWithError(QLatin1String("Cert.Untrusted"),
                             QLatin1String("Certificate rejected by the user"));
    }
}

bool TlsCertVerifierOp::verifyCertChain(const QCA::CertificateChain& chain)
{
    const QList<QSslCertificate> primary = QSslCertificate::fromData(chain.primary().toDER(), QSsl::Der);
    KSslCertificateManager *const cm = KSslCertificateManager::self();
    KSslCertificateRule rule = cm->rule(primary.first(), m_hostname);

    // Find all errors then are not ignored by the rule
    QList<KSslError> errors;
    Q_FOREACH(const QCA::Certificate &cert, chain) {
        QCA::Validity validity = cert.validate(CACollection(), QCA::CertificateCollection(), QCA::UsageTLSClient, QCA::ValidateAll);
        if (validity == QCA::ValidityGood) {
            continue;
        }

        KSslError::Error error = validityToError(validity);
        if (!rule.ignoredErrors().contains(error)) {
            errors << KSslError(error);
        }
    }

    // If all errors are ignored, just accept
    if (errors.isEmpty()) {
        return true;
    }

    QString message = i18n("The server failed the authenticity check (%1).\n\n", m_hostname);
    Q_FOREACH(const KSslError &error, errors) {
        message.append(error.errorString());
        message.append(QLatin1Char('\n'));
    }

    int msgResult;
    do {
        msgResult = KMessageBox::warningYesNoCancel(0,
                               message,
                               i18n("Server Authentication"),
                               KGuiItem(i18n("&Details"), QLatin1String("dialog-information")), // yes
                               KGuiItem(i18n("Co&ntinue"), QLatin1String("arrow-right")), // no
                               KGuiItem(i18n("&Cancel"), QLatin1String("dialog-cancel")));
        if (msgResult == KMessageBox::Yes) {
            showSslDialog(chain, errors);
        } else if (msgResult == KMessageBox::Cancel) {
            return false; // reject
        }
        // Fall through on KMessageBox::No
    } while (msgResult == KMessageBox::Yes);

    // Save the user's choice to ignore the SSL errors.
    msgResult = KMessageBox::warningYesNo(
                            0,
                            i18n("Would you like to accept this "
                                 "certificate forever without "
                                 "being prompted?"),
                            i18n("Server Authentication"),
                            KGuiItem(i18n("&Forever"), QLatin1String("flag-green")),
                            KGuiItem(i18n("&Current Session only"), QLatin1String("chronometer")));
    QDateTime ruleExpiry = QDateTime::currentDateTime();
    if (msgResult == KMessageBox::Yes) {
        // Accept forever ("for a very long time")
        ruleExpiry = ruleExpiry.addYears(1000);
    } else {
        // Accept "for a short time", half an hour.
        ruleExpiry = ruleExpiry.addSecs(30*60);
    }

    rule.setExpiryDateTime(ruleExpiry);
    rule.setIgnoredErrors(errors);
    cm->setRule(rule);

    return true;
}

void TlsCertVerifierOp::showSslDialog(const QCA::CertificateChain &chain, const QList<KSslError> &errors) const
{
    QString errorStr;
    Q_FOREACH (const KSslError &error, errors) {
        errorStr += QString::number(static_cast<int>(error.error())) + QLatin1Char('\t');
        errorStr += QLatin1Char('\n');
    }
    errorStr.chop(1);

    // No way to tell whether QSsl::TlsV1 or QSsl::TlsV1Ssl3
    KSslCipher cipher = QSslCipher(QLatin1String("TLS"), QSsl::TlsV1);
    QString sslCipher = cipher.encryptionMethod() + QLatin1Char('\n');
    sslCipher += cipher.authenticationMethod() + QLatin1Char('\n');
    sslCipher += cipher.keyExchangeMethod() + QLatin1Char('\n');
    sslCipher += cipher.digestMethod();

    const QList<QSslCertificate> qchain = chainToList(chain);
    QPointer<KSslInfoDialog> dialog(new KSslInfoDialog(0));
    dialog->setSslInfo(qchain,
                    QString(), // we don't know the IP
                    m_hostname, // the URL
                    QLatin1String("TLS"),
                    sslCipher,
                    cipher.usedBits(),
                    cipher.supportedBits(),
                    KSslInfoDialog::errorsFromString(errorStr));

    dialog->exec();
    delete dialog;
}

KSslError::Error TlsCertVerifierOp::validityToError(QCA::Validity validity) const
{
    switch (validity) {
        case QCA::ValidityGood:
            return KSslError::NoError;
        case QCA::ErrorRejected:
            return KSslError::RejectedCertificate;
        case QCA::ErrorUntrusted:
            return KSslError::UntrustedCertificate;
        case QCA::ErrorSignatureFailed:
            return KSslError::CertificateSignatureFailed;
        case QCA::ErrorInvalidCA:
            return KSslError::InvalidCertificateAuthorityCertificate;
        case QCA::ErrorInvalidPurpose:
            return KSslError::InvalidCertificatePurpose;
        case QCA::ErrorSelfSigned:
            return KSslError::SelfSignedCertificate;
        case QCA::ErrorRevoked:
            return KSslError::RevokedCertificate;
        case QCA::ErrorPathLengthExceeded:
            return KSslError::PathLengthExceeded;
        case QCA::ErrorExpired: // fall-through
        case QCA::ErrorExpiredCA:
            return KSslError::ExpiredCertificate;
        case QCA::ErrorValidityUnknown:
            return KSslError::UnknownError;
    }

    return KSslError::UnknownError;
}

QCA::CertificateCollection TlsCertVerifierOp::CACollection() const
{
    QList<QSslCertificate> certs = KSslCertificateManager::self()->caCertificates();
    QCA::CertificateCollection collection;

    Q_FOREACH(const QSslCertificate &cert, certs) {
        collection.addCertificate(QCA::Certificate::fromDER(cert.toDer()));
    }

    return collection;
}

QList< QSslCertificate > TlsCertVerifierOp::chainToList(const QCA::CertificateChain& chain) const
{
    QList<QSslCertificate> certs;
    Q_FOREACH(const QCA::Certificate &cert, chain) {
        certs << QSslCertificate::fromData(cert.toDER(), QSsl::Der);
    }

    return certs;
}

#include "tls-cert-verifier-op.moc"
<commit_msg>Fixed wrong compare causing segfault<commit_after>/*
 * Copyright (C) 2011 Collabora Ltd. <http://www.collabora.co.uk/>
 *   @author Andre Moreira Magalhaes <andre.magalhaes@collabora.co.uk>
 * Copyright (C) 2013 Dan Vrátil <dvratil@redhat.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */

#include "tls-cert-verifier-op.h"

#include <TelepathyQt/PendingVariantMap>

#include <KMessageBox>
#include <KLocalizedString>
#include <KDebug>

#include <QSslCertificate>
#include <QSslCipher>

#include "kssl/ksslcertificatemanager.h"
#include "kssl/ksslinfodialog.h"

#include <QtCrypto/QtCrypto>

TlsCertVerifierOp::TlsCertVerifierOp(const Tp::AccountPtr &account,
        const Tp::ConnectionPtr &connection,
        const Tp::ChannelPtr &channel)
    : Tp::PendingOperation(channel),
      m_account(account),
      m_connection(connection),
      m_channel(channel)
{
    QDBusObjectPath certificatePath = qdbus_cast<QDBusObjectPath>(channel->immutableProperties().value(
                TP_QT_IFACE_CHANNEL_TYPE_SERVER_TLS_CONNECTION + QLatin1String(".ServerCertificate")));
    m_hostname = qdbus_cast<QString>(channel->immutableProperties().value(
                TP_QT_IFACE_CHANNEL_TYPE_SERVER_TLS_CONNECTION + QLatin1String(".Hostname")));
    m_referenceIdentities = qdbus_cast<QStringList>(channel->immutableProperties().value(
                TP_QT_IFACE_CHANNEL_TYPE_SERVER_TLS_CONNECTION + QLatin1String(".ReferenceIdentities")));

    m_authTLSCertificateIface = new Tp::Client::AuthenticationTLSCertificateInterface(
            channel->dbusConnection(), channel->busName(), certificatePath.path());
    connect(m_authTLSCertificateIface->requestAllProperties(),
            SIGNAL(finished(Tp::PendingOperation*)),
            SLOT(gotProperties(Tp::PendingOperation*)));
}

TlsCertVerifierOp::~TlsCertVerifierOp()
{
}

void TlsCertVerifierOp::gotProperties(Tp::PendingOperation *op)
{
    if (op->isError()) {
        kWarning() << "Unable to retrieve properties from AuthenticationTLSCertificate object at" <<
            m_authTLSCertificateIface->path();
        m_channel->requestClose();
        setFinishedWithError(op->errorName(), op->errorMessage());
        return;
    }

    // everything ok, we can return from handleChannels now
    Q_EMIT ready(this);

    Tp::PendingVariantMap *pvm = qobject_cast<Tp::PendingVariantMap*>(op);
    QVariantMap props = qdbus_cast<QVariantMap>(pvm->result());
    m_certType = qdbus_cast<QString>(props.value(QLatin1String("CertificateType")));
    m_certData = qdbus_cast<CertificateDataList>(props.value(QLatin1String("CertificateChainData")));

    if(m_certType.compare(QLatin1String("\"x509\""), Qt::CaseInsensitive)) {
        Tp::TLSCertificateRejectionList rejections;
        m_authTLSCertificateIface->Reject(rejections);
        m_channel->requestClose();
        setFinishedWithError(QLatin1String("Cert.Unknown"),
                             QLatin1String("Invalid certificate type"));
        return;
    }

    // Initialize QCA module
    QCA::Initializer initializer;

    QCA::CertificateChain chain;
    Q_FOREACH (const QByteArray &data, m_certData) {
       chain << QCA::Certificate::fromDER(data);
    }

    if (verifyCertChain(chain)) {
        m_authTLSCertificateIface->Accept().waitForFinished();
        setFinished();
    } else {
        Tp::TLSCertificateRejectionList rejections;
        m_authTLSCertificateIface->Reject(rejections);
        m_channel->requestClose();
        setFinishedWithError(QLatin1String("Cert.Untrusted"),
                             QLatin1String("Certificate rejected by the user"));
    }
}

bool TlsCertVerifierOp::verifyCertChain(const QCA::CertificateChain& chain)
{
    const QList<QSslCertificate> primary = QSslCertificate::fromData(chain.primary().toDER(), QSsl::Der);
    KSslCertificateManager *const cm = KSslCertificateManager::self();
    KSslCertificateRule rule = cm->rule(primary.first(), m_hostname);

    // Find all errors then are not ignored by the rule
    QList<KSslError> errors;
    Q_FOREACH(const QCA::Certificate &cert, chain) {
        QCA::Validity validity = cert.validate(CACollection(), QCA::CertificateCollection(), QCA::UsageTLSClient, QCA::ValidateAll);
        if (validity == QCA::ValidityGood) {
            continue;
        }

        KSslError::Error error = validityToError(validity);
        if (!rule.ignoredErrors().contains(error)) {
            errors << KSslError(error);
        }
    }

    // If all errors are ignored, just accept
    if (errors.isEmpty()) {
        return true;
    }

    QString message = i18n("The server failed the authenticity check (%1).\n\n", m_hostname);
    Q_FOREACH(const KSslError &error, errors) {
        message.append(error.errorString());
        message.append(QLatin1Char('\n'));
    }

    int msgResult;
    do {
        msgResult = KMessageBox::warningYesNoCancel(0,
                               message,
                               i18n("Server Authentication"),
                               KGuiItem(i18n("&Details"), QLatin1String("dialog-information")), // yes
                               KGuiItem(i18n("Co&ntinue"), QLatin1String("arrow-right")), // no
                               KGuiItem(i18n("&Cancel"), QLatin1String("dialog-cancel")));
        if (msgResult == KMessageBox::Yes) {
            showSslDialog(chain, errors);
        } else if (msgResult == KMessageBox::Cancel) {
            return false; // reject
        }
        // Fall through on KMessageBox::No
    } while (msgResult == KMessageBox::Yes);

    // Save the user's choice to ignore the SSL errors.
    msgResult = KMessageBox::warningYesNo(
                            0,
                            i18n("Would you like to accept this "
                                 "certificate forever without "
                                 "being prompted?"),
                            i18n("Server Authentication"),
                            KGuiItem(i18n("&Forever"), QLatin1String("flag-green")),
                            KGuiItem(i18n("&Current Session only"), QLatin1String("chronometer")));
    QDateTime ruleExpiry = QDateTime::currentDateTime();
    if (msgResult == KMessageBox::Yes) {
        // Accept forever ("for a very long time")
        ruleExpiry = ruleExpiry.addYears(1000);
    } else {
        // Accept "for a short time", half an hour.
        ruleExpiry = ruleExpiry.addSecs(30*60);
    }

    rule.setExpiryDateTime(ruleExpiry);
    rule.setIgnoredErrors(errors);
    cm->setRule(rule);

    return true;
}

void TlsCertVerifierOp::showSslDialog(const QCA::CertificateChain &chain, const QList<KSslError> &errors) const
{
    QString errorStr;
    Q_FOREACH (const KSslError &error, errors) {
        errorStr += QString::number(static_cast<int>(error.error())) + QLatin1Char('\t');
        errorStr += QLatin1Char('\n');
    }
    errorStr.chop(1);

    // No way to tell whether QSsl::TlsV1 or QSsl::TlsV1Ssl3
    KSslCipher cipher = QSslCipher(QLatin1String("TLS"), QSsl::TlsV1);
    QString sslCipher = cipher.encryptionMethod() + QLatin1Char('\n');
    sslCipher += cipher.authenticationMethod() + QLatin1Char('\n');
    sslCipher += cipher.keyExchangeMethod() + QLatin1Char('\n');
    sslCipher += cipher.digestMethod();

    const QList<QSslCertificate> qchain = chainToList(chain);
    QPointer<KSslInfoDialog> dialog(new KSslInfoDialog(0));
    dialog->setSslInfo(qchain,
                    QString(), // we don't know the IP
                    m_hostname, // the URL
                    QLatin1String("TLS"),
                    sslCipher,
                    cipher.usedBits(),
                    cipher.supportedBits(),
                    KSslInfoDialog::errorsFromString(errorStr));

    dialog->exec();
    delete dialog;
}

KSslError::Error TlsCertVerifierOp::validityToError(QCA::Validity validity) const
{
    switch (validity) {
        case QCA::ValidityGood:
            return KSslError::NoError;
        case QCA::ErrorRejected:
            return KSslError::RejectedCertificate;
        case QCA::ErrorUntrusted:
            return KSslError::UntrustedCertificate;
        case QCA::ErrorSignatureFailed:
            return KSslError::CertificateSignatureFailed;
        case QCA::ErrorInvalidCA:
            return KSslError::InvalidCertificateAuthorityCertificate;
        case QCA::ErrorInvalidPurpose:
            return KSslError::InvalidCertificatePurpose;
        case QCA::ErrorSelfSigned:
            return KSslError::SelfSignedCertificate;
        case QCA::ErrorRevoked:
            return KSslError::RevokedCertificate;
        case QCA::ErrorPathLengthExceeded:
            return KSslError::PathLengthExceeded;
        case QCA::ErrorExpired: // fall-through
        case QCA::ErrorExpiredCA:
            return KSslError::ExpiredCertificate;
        case QCA::ErrorValidityUnknown:
            return KSslError::UnknownError;
    }

    return KSslError::UnknownError;
}

QCA::CertificateCollection TlsCertVerifierOp::CACollection() const
{
    QList<QSslCertificate> certs = KSslCertificateManager::self()->caCertificates();
    QCA::CertificateCollection collection;

    Q_FOREACH(const QSslCertificate &cert, certs) {
        collection.addCertificate(QCA::Certificate::fromDER(cert.toDer()));
    }

    return collection;
}

QList< QSslCertificate > TlsCertVerifierOp::chainToList(const QCA::CertificateChain& chain) const
{
    QList<QSslCertificate> certs;
    Q_FOREACH(const QCA::Certificate &cert, chain) {
        certs << QSslCertificate::fromData(cert.toDER(), QSsl::Der);
    }

    return certs;
}

#include "tls-cert-verifier-op.moc"
<|endoftext|>


<commit_before>#include <QApplication>
#include <QFileInfo>
#include "player.h"
#include "config.h"
#include "tokolosh_interface.h"

static inline QFileInfo resolveSymlink(const QString &file)
{
    QFileInfo fi(file);
    QStringList paths;
    forever {
        if (paths.contains(fi.absoluteFilePath())) {
            qWarning("%s is a recursive symlink", qPrintable(paths.join(" => ")));
            return QFileInfo();
        }
        paths.append(fi.absoluteFilePath());
        if (!fi.isSymLink()) {
            break;
        }
        fi = fi.symLinkTarget();
    }
    return fi;
}

static inline QStringList toFiles(const QString &arg, bool recursive = false)
{
    const QFileInfo fi = ::resolveSymlink(arg);
    QStringList files;
    if (fi.isDir()) {
        const QDir dir(fi.absoluteFilePath());
        foreach(QString file, dir.entryList(QDir::Dirs|QDir::Files|QDir::NoDotAndDotDot)) {
            const QFileInfo f = ::resolveSymlink(file);
            if (!f.isDir()) {
                files.append(f.absoluteFilePath());
            } else if (recursive) {
                files += ::toFiles(f.absoluteFilePath(), recursive);
            }
        }
    } else if (fi.isFile()) {
        files.append(fi.absoluteFilePath());
    }
    return files;
}

static inline QMetaMethod findMethod(QString arg, const QMetaObject *metaObject)
{
    QRegExp rx("^-*");
    arg.remove(rx);
    const int methodCount = metaObject->methodCount();
    for (int i=metaObject->methodOffset(); i<methodCount; ++i) {
        const QMetaMethod method = metaObject->method(i);
        if (method.methodType() != QMetaMethod::Slot
            && method.methodType() != QMetaMethod::Method)
            continue;

        const QString methodName = method.signature();
        if (!methodName.startsWith(arg))
            continue;

        if (methodName.size() != arg.size()) {
            // if the argument part is not specified that's still a match
            const int index = methodName.lastIndexOf("(");
            if (arg.size() < index) // if you specify setVo that shouldn't match setVolume(or should it?)
                continue;
        }
        return method;
    }
    return QMetaMethod();
}

Q_DECLARE_METATYPE(QDBusReply<void>);
Q_DECLARE_METATYPE(QDBusReply<int>);
Q_DECLARE_METATYPE(QDBusReply<bool>);
Q_DECLARE_METATYPE(QDBusReply<QString>);
Q_DECLARE_METATYPE(QDBusReply<double>);
Q_DECLARE_METATYPE(QDBusReply<QTime>);
Q_DECLARE_METATYPE(QDBusReply<QDateTime>);
Q_DECLARE_METATYPE(QDBusReply<QDate>);
Q_DECLARE_METATYPE(QDBusReply<QStringList>);
Q_DECLARE_METATYPE(QDBusReply<QRegExp>);

static QHash<int, int> registerMetaTypes()
{
    static QHash<int, int> types;
    if (types.isEmpty()) {
        types[qRegisterMetaType<QDBusReply<void> >("QDBusReply<void>")] = QMetaType::Void;
        types[qRegisterMetaType<QDBusReply<int> >("QDBusReply<int>")] = QMetaType::Int;
        types[qRegisterMetaType<QDBusReply<bool> >("QDBusReply<bool>")] = QMetaType::Bool;
        types[qRegisterMetaType<QDBusReply<QString> >("QDBusReply<QString>")] = QMetaType::QString;
        types[qRegisterMetaType<QDBusReply<double> >("QDBusReply<double>")] = QMetaType::Double;
        types[qRegisterMetaType<QDBusReply<QTime> >("QDBusReply<QTime>")] = QMetaType::QTime;
        types[qRegisterMetaType<QDBusReply<QDateTime> >("QDBusReply<QDateTime>")] = QMetaType::QDateTime;
        types[qRegisterMetaType<QDBusReply<QDate> >("QDBusReply<QDate>")] = QMetaType::QDate;
        types[qRegisterMetaType<QDBusReply<QStringList> >("QDBusReply<QStringList>")] = QMetaType::QStringList;
        types[qRegisterMetaType<QDBusReply<QRegExp> >("QDBusReply<QRegExp>")] = QMetaType::QRegExp;
    }
    return types;
}

static QVariant toVariant(const QVariant &dbusReply)
{
    const QHash<int, int> types = registerMetaTypes();
    const int type = types.value(dbusReply.userType());
    switch (type) {
    case QMetaType::Void: return QVariant();
    case QMetaType::Int: return qVariantValue<QDBusReply<int> >(dbusReply).value();
    case QMetaType::Bool: return qVariantValue<QDBusReply<bool> >(dbusReply).value();
    case QMetaType::QString: return qVariantValue<QDBusReply<QString> >(dbusReply).value();
    case QMetaType::Double: return qVariantValue<QDBusReply<double> >(dbusReply).value();
    case QMetaType::QTime: return qVariantValue<QDBusReply<QTime> >(dbusReply).value();
    case QMetaType::QDateTime: return qVariantValue<QDBusReply<QDateTime> >(dbusReply).value();
    case QMetaType::QDate: return qVariantValue<QDBusReply<QDate> >(dbusReply).value();
    case QMetaType::QStringList: return qVariantValue<QDBusReply<QStringList> >(dbusReply).value();
    case QMetaType::QRegExp: return qVariantValue<QDBusReply<QRegExp> >(dbusReply).value();
    default: qWarning("Unknown type %d %d", dbusReply.userType(), types.value(dbusReply.userType()));
    }
    return QVariant();
}

int main(int argc, char *argv[])
{
    const QHash<int, int> types = registerMetaTypes();
    TokoloshInterface dbusInterface("com.TokoloshXineBackend.TokoloshMediaPlayer",
                                    "/TokoloshMediaPlayer",
                                    QDBusConnection::sessionBus());
    if (argc > 1) {
        QCoreApplication *coreApp = new QCoreApplication(argc, argv);
        coreApp->setApplicationName("tokoloshhead_v2");

        const QMetaObject *dbusInterfaceMetaObject = dbusInterface.metaObject();
        const QStringList args = coreApp->arguments();
        for (int i=1; i<argc; ++i) {
            const QString &arg = args.at(i);
            if (arg == "--list-methods") {
                for (int j=dbusInterfaceMetaObject->methodOffset(); j<dbusInterfaceMetaObject->methodCount(); ++j) {
                    printf("%s\n", dbusInterfaceMetaObject->method(j).signature());
                }
                return 0;
            }
            const QMetaMethod method = ::findMethod(arg, dbusInterfaceMetaObject);
            if (!method.signature())
                continue;

            QVariant returnArg(static_cast<QVariant::Type>(QMetaType::type(method.typeName())));
            const QList<QByteArray> parameterTypes = method.parameterTypes();
            bool ret = false;
            if (parameterTypes.isEmpty()) {
                ret = method.invoke(&dbusInterface, Qt::DirectConnection,
                                    QGenericReturnArgument(returnArg.typeName(), returnArg.data()));
            } else if (argc - i - 1 < parameterTypes.size()) {
                qWarning("Not enough arguments specified for %s needed %d, got %d",
                         method.signature(), parameterTypes.size(), argc - i - 1);
                return false; // ### ???
            } else {
                QVariant arguments[10];
                for (int j=0; j<parameterTypes.size(); ++j) {
                    const int type = QMetaType::type(parameterTypes.at(j).constData());
                    arguments[j] = args.at(++i);
                    if (!arguments[j].convert(static_cast<QVariant::Type>(type))) {
                        qWarning("Can't convert %s to %s", qPrintable(args.at(i)), parameterTypes.at(i).constData());
                        return false; // ### ???
                    }
                }
                ret = method.invoke(&dbusInterface, Qt::DirectConnection,
                                    QGenericReturnArgument(returnArg.typeName(), returnArg.data()),
                                    QGenericArgument(parameterTypes.value(0).constData(), arguments[0].data()),
                                    QGenericArgument(parameterTypes.value(1).constData(), arguments[1].data()),
                                    QGenericArgument(parameterTypes.value(2).constData(), arguments[2].data()),
                                    QGenericArgument(parameterTypes.value(3).constData(), arguments[3].data()),
                                    QGenericArgument(parameterTypes.value(4).constData(), arguments[4].data()),
                                    QGenericArgument(parameterTypes.value(5).constData(), arguments[5].data()),
                                    QGenericArgument(parameterTypes.value(6).constData(), arguments[6].data()),
                                    QGenericArgument(parameterTypes.value(7).constData(), arguments[7].data()),
                                    QGenericArgument(parameterTypes.value(8).constData(), arguments[8].data()),
                                    QGenericArgument(parameterTypes.value(9).constData(), arguments[9].data()));
            }
            if (!ret) {
                qWarning("Can't invoke %s", method.signature());
            } else {
                QFile f;
                f.open(stdout, QIODevice::WriteOnly);
                QDebug out(&f);
                const QVariant var = ::toVariant(returnArg);
                out << "Invoked" << method.signature() << "successfully" << returnArg << var << endl;
                return 0;
            }
        }
        delete coreApp;
    }
//             }

//             if (QFileInfo(consideredPath).isFile()) {
//                 dbusInterface.load(arg);
//             } else {
//                 //handle me
//                 //path loading not in interface! investigating seperately
//             }
//         } else if (arg.startsWith('-')) {
//             queuedMethodIndex = -1;
//             QByteArray sig = arg;
//             const char* charArg = arg.endsWith("()")
//                                   ? arg.toAscii().constData()
//                                   : arg.append("()").toAscii().constData();
//             //char*+1 is a nasty but efficient way to drop the first char
//             const int index = dbusInterfaceMetaObject->indexOfSlot(charArg+1);
//             if (index != -1) {
//                 qWarning("%s has index %d",charArg+1,index);
//                 QMetaMethod calledMethod = dbusInterfaceMetaObject->method(index);
//                 if (calledMethod.parameterNames().size() > 0) {
//                     //gag for your arg beatch
//                     queuedMethodIndex = index;
//                 } else {
//                     calledMethod.invoke(&dbusInterface,
//                                         Qt::DirectConnection,
//                                         QGenericReturnArgument());

//                     app.quit();
//                     return 0;
//                 }
//             } else {
//                 qWarning("Unknown argument %s", qPrintable(arg));
//             }
//         } else {
//             if (queuedMethodIndex!=-1) {
//                 QMetaMethod calledMethod = dbusInterfaceMetaObject->method(queuedMethodIndex);
//                 calledMethod.invoke(&dbusInterface,
//                                     Qt::DirectConnection,
//                                     QGenericReturnArgument(),
//                                     Q_ARG(QString, arg));

//                 app.quit();
//                 return 0;
//             }
//             qWarning("%s doesn't seem to exist", qPrintable(arg));
//             queuedMethodIndex = -1;
//         }
//     }
    QApplication app(argc, argv);
    app.setApplicationName("tokoloshhead_v2");
    Player player(&dbusInterface);
    if (!player.setSkin(Config::value<QString>("skin", QString(":/skins/dullSod")))) {
        const bool ret = player.setSkin(QLatin1String(":/skins/dullSod"));
        Q_ASSERT(ret);
        Q_UNUSED(ret);
    }
    player.show();
    return app.exec();
}
<commit_msg>add QList<int> as a possible value<commit_after>#include <QApplication>
#include <QFileInfo>
#include "player.h"
#include "config.h"
#include "tokolosh_interface.h"

static inline QFileInfo resolveSymlink(const QString &file)
{
    QFileInfo fi(file);
    QStringList paths;
    forever {
        if (paths.contains(fi.absoluteFilePath())) {
            qWarning("%s is a recursive symlink", qPrintable(paths.join(" => ")));
            return QFileInfo();
        }
        paths.append(fi.absoluteFilePath());
        if (!fi.isSymLink()) {
            break;
        }
        fi = fi.symLinkTarget();
    }
    return fi;
}

static inline QStringList toFiles(const QString &arg, bool recursive = false)
{
    const QFileInfo fi = ::resolveSymlink(arg);
    QStringList files;
    if (fi.isDir()) {
        const QDir dir(fi.absoluteFilePath());
        foreach(QString file, dir.entryList(QDir::Dirs|QDir::Files|QDir::NoDotAndDotDot)) {
            const QFileInfo f = ::resolveSymlink(file);
            if (!f.isDir()) {
                files.append(f.absoluteFilePath());
            } else if (recursive) {
                files += ::toFiles(f.absoluteFilePath(), recursive);
            }
        }
    } else if (fi.isFile()) {
        files.append(fi.absoluteFilePath());
    }
    return files;
}

static inline QMetaMethod findMethod(QString arg, const QMetaObject *metaObject)
{
    QRegExp rx("^-*");
    arg.remove(rx);
    const int methodCount = metaObject->methodCount();
    for (int i=metaObject->methodOffset(); i<methodCount; ++i) {
        const QMetaMethod method = metaObject->method(i);
        if (method.methodType() != QMetaMethod::Slot
            && method.methodType() != QMetaMethod::Method)
            continue;

        const QString methodName = method.signature();
        if (!methodName.startsWith(arg))
            continue;

        if (methodName.size() != arg.size()) {
            // if the argument part is not specified that's still a match
            const int index = methodName.lastIndexOf("(");
            if (arg.size() < index) // if you specify setVo that shouldn't match setVolume(or should it?)
                continue;
        }
        return method;
    }
    return QMetaMethod();
}

Q_DECLARE_METATYPE(QDBusReply<void>);
Q_DECLARE_METATYPE(QDBusReply<int>);
Q_DECLARE_METATYPE(QDBusReply<bool>);
Q_DECLARE_METATYPE(QDBusReply<QString>);
Q_DECLARE_METATYPE(QDBusReply<double>);
Q_DECLARE_METATYPE(QDBusReply<QTime>);
Q_DECLARE_METATYPE(QDBusReply<QDateTime>);
Q_DECLARE_METATYPE(QDBusReply<QDate>);
Q_DECLARE_METATYPE(QDBusReply<QStringList>);
Q_DECLARE_METATYPE(QDBusReply<QRegExp>);
Q_DECLARE_METATYPE(QDBusReply<QList<int> >);

static QHash<int, int> registerMetaTypes()
{
    static QHash<int, int> types;
    if (types.isEmpty()) {
        types[qRegisterMetaType<QDBusReply<void> >("QDBusReply<void>")] = QMetaType::Void;
        types[qRegisterMetaType<QDBusReply<int> >("QDBusReply<int>")] = QMetaType::Int;
        types[qRegisterMetaType<QDBusReply<bool> >("QDBusReply<bool>")] = QMetaType::Bool;
        types[qRegisterMetaType<QDBusReply<QString> >("QDBusReply<QString>")] = QMetaType::QString;
        types[qRegisterMetaType<QDBusReply<double> >("QDBusReply<double>")] = QMetaType::Double;
        types[qRegisterMetaType<QDBusReply<QTime> >("QDBusReply<QTime>")] = QMetaType::QTime;
        types[qRegisterMetaType<QDBusReply<QDateTime> >("QDBusReply<QDateTime>")] = QMetaType::QDateTime;
        types[qRegisterMetaType<QDBusReply<QDate> >("QDBusReply<QDate>")] = QMetaType::QDate;
        types[qRegisterMetaType<QDBusReply<QStringList> >("QDBusReply<QStringList>")] = QMetaType::QStringList;
        types[qRegisterMetaType<QDBusReply<QRegExp> >("QDBusReply<QRegExp>")] = QMetaType::QRegExp;
        types[qRegisterMetaType<QDBusReply<QRegExp> >("QDBusReply<QList<int> >")] = QMetaType::QRegExp;
    }
    return types;
}

static QVariant toVariant(const QVariant &dbusReply)
{
    const QHash<int, int> types = registerMetaTypes();
    const int type = types.value(dbusReply.userType());
    switch (type) {
    case QMetaType::Void: return QVariant();
    case QMetaType::Int: return qVariantValue<QDBusReply<int> >(dbusReply).value();
    case QMetaType::Bool: return qVariantValue<QDBusReply<bool> >(dbusReply).value();
    case QMetaType::QString: return qVariantValue<QDBusReply<QString> >(dbusReply).value();
    case QMetaType::Double: return qVariantValue<QDBusReply<double> >(dbusReply).value();
    case QMetaType::QTime: return qVariantValue<QDBusReply<QTime> >(dbusReply).value();
    case QMetaType::QDateTime: return qVariantValue<QDBusReply<QDateTime> >(dbusReply).value();
    case QMetaType::QDate: return qVariantValue<QDBusReply<QDate> >(dbusReply).value();
    case QMetaType::QStringList: return qVariantValue<QDBusReply<QStringList> >(dbusReply).value();
    case QMetaType::QRegExp: return qVariantValue<QDBusReply<QRegExp> >(dbusReply).value();
    default: qWarning("Unknown type %d %d", dbusReply.userType(), types.value(dbusReply.userType()));
    }
    return QVariant();
}

int main(int argc, char *argv[])
{
    const QHash<int, int> types = registerMetaTypes();
    TokoloshInterface dbusInterface("com.TokoloshXineBackend.TokoloshMediaPlayer",
                                    "/TokoloshMediaPlayer",
                                    QDBusConnection::sessionBus());
    if (argc > 1) {
        QCoreApplication *coreApp = new QCoreApplication(argc, argv);
        coreApp->setApplicationName("tokoloshhead_v2");

        const QMetaObject *dbusInterfaceMetaObject = dbusInterface.metaObject();
        const QStringList args = coreApp->arguments();
        for (int i=1; i<argc; ++i) {
            const QString &arg = args.at(i);
            if (arg == "--list-methods") {
                for (int j=dbusInterfaceMetaObject->methodOffset(); j<dbusInterfaceMetaObject->methodCount(); ++j) {
                    printf("%s\n", dbusInterfaceMetaObject->method(j).signature());
                }
                return 0;
            }
            const QMetaMethod method = ::findMethod(arg, dbusInterfaceMetaObject);
            if (!method.signature())
                continue;

            QVariant returnArg(static_cast<QVariant::Type>(QMetaType::type(method.typeName())));
            const QList<QByteArray> parameterTypes = method.parameterTypes();
            bool ret = false;
            if (parameterTypes.isEmpty()) {
                ret = method.invoke(&dbusInterface, Qt::DirectConnection,
                                    QGenericReturnArgument(returnArg.typeName(), returnArg.data()));
            } else if (argc - i - 1 < parameterTypes.size()) {
                qWarning("Not enough arguments specified for %s needed %d, got %d",
                         method.signature(), parameterTypes.size(), argc - i - 1);
                return false; // ### ???
            } else {
                QVariant arguments[10];
                for (int j=0; j<parameterTypes.size(); ++j) {
                    const int type = QMetaType::type(parameterTypes.at(j).constData());
                    arguments[j] = args.at(++i);
                    if (!arguments[j].convert(static_cast<QVariant::Type>(type))) {
                        qWarning("Can't convert %s to %s", qPrintable(args.at(i)), parameterTypes.at(i).constData());
                        return false; // ### ???
                    }
                }
                ret = method.invoke(&dbusInterface, Qt::DirectConnection,
                                    QGenericReturnArgument(returnArg.typeName(), returnArg.data()),
                                    QGenericArgument(parameterTypes.value(0).constData(), arguments[0].data()),
                                    QGenericArgument(parameterTypes.value(1).constData(), arguments[1].data()),
                                    QGenericArgument(parameterTypes.value(2).constData(), arguments[2].data()),
                                    QGenericArgument(parameterTypes.value(3).constData(), arguments[3].data()),
                                    QGenericArgument(parameterTypes.value(4).constData(), arguments[4].data()),
                                    QGenericArgument(parameterTypes.value(5).constData(), arguments[5].data()),
                                    QGenericArgument(parameterTypes.value(6).constData(), arguments[6].data()),
                                    QGenericArgument(parameterTypes.value(7).constData(), arguments[7].data()),
                                    QGenericArgument(parameterTypes.value(8).constData(), arguments[8].data()),
                                    QGenericArgument(parameterTypes.value(9).constData(), arguments[9].data()));
            }
            if (!ret) {
                qWarning("Can't invoke %s", method.signature());
            } else {
                QFile f;
                f.open(stdout, QIODevice::WriteOnly);
                QDebug out(&f);
                const QVariant var = ::toVariant(returnArg);
                out << "Invoked" << method.signature() << "successfully" << returnArg << var << endl;
                return 0;
            }
        }
        delete coreApp;
    }
//             }

//             if (QFileInfo(consideredPath).isFile()) {
//                 dbusInterface.load(arg);
//             } else {
//                 //handle me
//                 //path loading not in interface! investigating seperately
//             }
//         } else if (arg.startsWith('-')) {
//             queuedMethodIndex = -1;
//             QByteArray sig = arg;
//             const char* charArg = arg.endsWith("()")
//                                   ? arg.toAscii().constData()
//                                   : arg.append("()").toAscii().constData();
//             //char*+1 is a nasty but efficient way to drop the first char
//             const int index = dbusInterfaceMetaObject->indexOfSlot(charArg+1);
//             if (index != -1) {
//                 qWarning("%s has index %d",charArg+1,index);
//                 QMetaMethod calledMethod = dbusInterfaceMetaObject->method(index);
//                 if (calledMethod.parameterNames().size() > 0) {
//                     //gag for your arg beatch
//                     queuedMethodIndex = index;
//                 } else {
//                     calledMethod.invoke(&dbusInterface,
//                                         Qt::DirectConnection,
//                                         QGenericReturnArgument());

//                     app.quit();
//                     return 0;
//                 }
//             } else {
//                 qWarning("Unknown argument %s", qPrintable(arg));
//             }
//         } else {
//             if (queuedMethodIndex!=-1) {
//                 QMetaMethod calledMethod = dbusInterfaceMetaObject->method(queuedMethodIndex);
//                 calledMethod.invoke(&dbusInterface,
//                                     Qt::DirectConnection,
//                                     QGenericReturnArgument(),
//                                     Q_ARG(QString, arg));

//                 app.quit();
//                 return 0;
//             }
//             qWarning("%s doesn't seem to exist", qPrintable(arg));
//             queuedMethodIndex = -1;
//         }
//     }
    QApplication app(argc, argv);
    app.setApplicationName("tokoloshhead_v2");
    Player player(&dbusInterface);
    if (!player.setSkin(Config::value<QString>("skin", QString(":/skins/dullSod")))) {
        const bool ret = player.setSkin(QLatin1String(":/skins/dullSod"));
        Q_ASSERT(ret);
        Q_UNUSED(ret);
    }
    player.show();
    return app.exec();
}
<|endoftext|>


<commit_before>#include<iostream>
#include<cmath>
#include <kv/interval.hpp>
#include <kv/rdouble.hpp>
#include<iomanip> // include this to set digits
using namespace std;
int main(void)
{
  // verfication program for the 2nd Jackson  q-Bessel function
  // properties of the q-Pochhammer symbol were used
  int K=2500;
  int j=1;
  kv::interval<double> q;
  kv::interval<double> nu; // nu is a real number
  kv::interval<double> x;
  q=0.1;// q must be positive
  nu=3.5;
  x=1.5;
  kv::interval<double> a;
  kv::interval<double> b;
  kv::interval<double> c;
  a=1.;
  b=1.;
  c=1.;
 // derived from the definition of q-Pochhammer symbol
  for (int k=1; k<=K; k++){
    j = -1*j;
    b=b*(1-pow(q,k));
    c=c*(1-pow(q,k+nu));
    a = a+j*pow(x/2,2*k)/(b*c);
  }
  b=b*(1-pow(q,K+1));
  c=c*(1-pow(q,K+nu+1));
  double rad=abs(pow(-1,K+1)*pow(x/2,2*K+2)/(b*c)).upper();
  // .upper() allows to output supremum
  kv::interval<double> series=a+rad; // result must be an interval

  kv::interval<double> z;
  z=pow(q,nu+1);
  kv::interval<double> f;
  kv::interval<double> g;
  f=1/(1-z);
  g=1.;
  int m=1;
 // derived from the definition of q-Pochhammer symbol
  for (int l=1; l<=K; l++){
    m = -1*m;
    g=g*(1-pow(q,l));
    f=f+m*pow(q,l*(l+1)/2)/(g*(1-z*pow(q,l)));
  }
  g=g*(1-pow(q,K+1));
  double rad2=abs(pow(-1,K+1)*pow(q,(K+1)*(K+2)/2)/(g*(1-z*pow(q,K+1)))).upper();
  kv::interval<double> denom=f+rad2; // result must be an interval
  kv::interval<double> first=pow(x/2,nu)*series/denom; // first OK
  // from below, Hahn`s formula will be used
  // this formula describes the relation between 1st and 2nd Jackson q-Bessel function
  // calculate ((-x^2)/4;q)_Infinity with q-exponential E_q(z)
  kv::interval<double> z1;
  z1=-pow(x,2)/4;
  kv::interval<double> d;
  kv::interval<double> e;
  d=1/(1-z1);
  e=1.;
  int j1;
  j1=1;
  // derived from the definition of q-Pochhammer symbol
  for (int n=1; n<=K; n++){
    j1 = -1*j1;
    e=e*(1-pow(q,n));
    d=d+j1*pow(q,n*(n+1)/2)/(e*(1-z1*pow(q,n)));
  }
  e=e*(1-pow(q,K+1));
  double rad3=abs(pow(-1,K+1)*pow(q,(K+1)*(K+2)/2)/(e*(1-z1*pow(q,K+1)))).upper();
  kv::interval<double> series2=d+rad3; // result must be an interval
  // series 2 OK
  // calculate the value of Euler function
  kv::interval<double> h=1-q;
  int m1=1;
  for (int r=1; r<=K; r++){
    m1 = -1*m1;
    h = h+m1*(1-pow(q,2*r+1))*pow(q,r*(3*r+1)/2);
  }
  double eulerrad=abs(pow(-1,K+1)*(1-pow(q,2*K+3))*pow(q,(K+1)*(3*K+4)/2)).upper();
  kv::interval<double> euler=h+eulerrad; // result must be an interval
  // euler OK
  kv::interval<double> hahn;
  hahn=euler/series2;
  // hahn OK

  kv::interval<double> res;
  res=first*hahn;
  cout << fixed << setprecision(16) <<res << endl;
}
// definition of the q-exponential function is given in the reference below
// Basic Hypergeometric Series Cambridge University Press (Gasper, Rahman, 1990)

// this program needs to be modified
<commit_msg>Delete Hahn-2nd-Jackson-q-Bessel.cc<commit_after><|endoftext|>


<commit_before>#include "dynet/nodes.h"
#include "dynet/dynet.h"
#include "dynet/training.h"
#include "dynet/timing.h"
#include "dynet/rnn.h"
#include "dynet/gru.h"
#include "dynet/lstm.h"
#include "dynet/fast-lstm.h"
#include "dynet/dict.h"
#include "dynet/expr.h"

#include <sys/stat.h>
#include <sys/types.h>
#include <iostream>
#include <fstream>
#include <sstream>
#include <type_traits>
#include <time.h>

#include <boost/serialization/vector.hpp>
#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <boost/program_options.hpp>

#include "s2s/encdec.hpp"
#include "s2s/decode.hpp"
#include "s2s/define.hpp"
#include "s2s/comp.hpp"
#include "s2s/metrics.hpp"
#include "s2s/options.hpp"

namespace s2s {

    void train(const s2s_options &opts){
        s2s::dicts dicts;
        s2s::parallel_corpus para_corp_train;
        s2s::parallel_corpus para_corp_dev;
        dicts.set(opts);
        para_corp_train.load_src(opts.srcfile, dicts);
        para_corp_train.load_trg(opts.trgfile, dicts);
        para_corp_train.load_check();
        para_corp_dev.load_src(opts.srcvalfile, dicts);
        para_corp_dev.load_trg(opts.trgvalfile, dicts);
        para_corp_dev.load_check();
        if(opts.guided_alignment == true){
            para_corp_train.load_align(opts.alignfile);
            para_corp_train.load_check_with_align();
            para_corp_dev.load_align(opts.alignvalfile);
            para_corp_dev.load_check_with_align();
        }
        // for debug
        dicts.save(opts);
        // for debug
        dynet::Model model;
        encoder_decoder* encdec = new encoder_decoder(model, &opts);
        encdec->enable_dropout();
        dynet::Trainer* trainer = nullptr;
        if(opts.optim == "sgd"){
            trainer = new dynet::SimpleSGDTrainer(model);
        }else if(opts.optim == "momentum_sgd"){
            trainer = new dynet::MomentumSGDTrainer(model);
        }else if(opts.optim == "adagrad"){
            trainer = new dynet::AdagradTrainer(model);
        }else if(opts.optim == "adadelta"){
            trainer = new dynet::AdadeltaTrainer(model);
        }else if(opts.optim == "adam"){
            trainer = new dynet::AdamTrainer(model);
        }else{
            std::cerr << "Trainer does not exist !"<< std::endl;
            assert(false);
        }
        float learning_rate = opts.learning_rate;
        trainer->eta0 = learning_rate;
        trainer->eta = learning_rate;
        trainer->eta_decay = 0.f;
        trainer->clip_threshold = opts.clip_threshold;
        trainer->clipping_enabled = opts.clipping_enabled;
        unsigned int epoch = 0;
        float align_w = opts.guided_alignment_weight;
        while(epoch < opts.epochs){
            // train
            para_corp_train.shuffle();
            batch one_batch;
            unsigned int bid = 0;
            while(para_corp_train.next_batch_para(one_batch, opts.max_batch_train, dicts)){
                bid++;
                one_batch.drop_word(dicts, opts);
                auto chrono_start = std::chrono::system_clock::now();
                dynet::ComputationGraph cg;
                std::vector<dynet::expr::Expression> errs_att;
                std::vector<dynet::expr::Expression> errs_out;
                float loss_att = 0.0;
                float loss_out = 0.0;
                std::vector<dynet::expr::Expression> i_enc = encdec->encoder(one_batch, cg);
                std::vector<dynet::expr::Expression> i_feed = encdec->init_feed(one_batch, cg);
                for (unsigned int t = 0; t < one_batch.trg.size() - 1; ++t) {
                    dynet::expr::Expression i_att_t = encdec->decoder_attention(cg, one_batch.trg[t], i_feed[t], i_enc[0]);
                    if(opts.guided_alignment == true){
                        for(unsigned int i = 0; i < one_batch.align.at(t).size(); i++){
                            assert(0 <= one_batch.align.at(t+1).at(i) < one_batch.src.size());
                        }
                        dynet::expr::Expression i_err = pickneglogsoftmax(i_att_t, one_batch.align.at(t+1));
                        errs_att.push_back(i_err);
                    }
                    std::vector<dynet::expr::Expression> i_out_t = encdec->decoder_output(cg, i_att_t, i_enc[1]);
                    i_feed.push_back(i_out_t[1]);
                    dynet::expr::Expression i_err = pickneglogsoftmax(i_out_t[0], one_batch.trg[t+1]);
                    errs_out.push_back(i_err);
                }
                dynet::expr::Expression i_nerr_out = sum_batches(sum(errs_out));
                loss_out = as_scalar(cg.forward(i_nerr_out));
                dynet::expr::Expression i_nerr_all;
                if(opts.guided_alignment == true){
                    dynet::expr::Expression i_nerr_att = sum_batches(sum(errs_att));
                    loss_att = as_scalar(cg.incremental_forward(i_nerr_att));
                    i_nerr_all = i_nerr_out + align_w * i_nerr_att;
                }else{
                    i_nerr_all = i_nerr_out;
                }
                cg.incremental_forward(i_nerr_all);
                cg.backward(i_nerr_all);
                //cg.print_graphviz();
                trainer->update(1.0 / double(one_batch.src.at(0).at(0).size()));
                auto chrono_end = std::chrono::system_clock::now();
                auto time_used = (double)std::chrono::duration_cast<std::chrono::milliseconds>(chrono_end - chrono_start).count() / (double)1000;
                std::cerr << "batch: " << bid;
                std::cerr << ",\tsize: " << one_batch.src.at(0).at(0).size();
                std::cerr << ",\toutput loss: " << loss_out;
                std::cerr << ",\tattention loss: " << loss_att;
                std::cerr << ",\tsource length: " << one_batch.src.size();
                std::cerr << ",\ttarget length: " << one_batch.trg.size();
                std::cerr << ",\ttime: " << time_used << " [s]" << std::endl;
                std::cerr << "[epoch=" << trainer->epoch << " eta=" << trainer->eta << " align_w=" << align_w << " clips=" << trainer->clips_since_status << " updates=" << trainer->updates_since_status << "] " << std::endl;
            }
            para_corp_train.reset_index();
            trainer->update_epoch();
            trainer->status();
            std::cerr << std::endl;
            // dev
            encdec->disable_dropout();
            std::cerr << "dev" << std::endl;
            ofstream dev_sents(opts.rootdir + "/dev_" + to_string(epoch) + ".txt");
            while(para_corp_dev.next_batch_para(one_batch, opts.max_batch_pred, dicts)){
                std::vector<std::vector<unsigned int> > osent;
                s2s::greedy_decode(one_batch, osent, encdec, dicts, opts);
                dev_sents << s2s::print_sents(osent, dicts);
            }
            dev_sents.close();
            para_corp_dev.reset_index();
            encdec->enable_dropout();
            // save Model
            ofstream model_out(opts.rootdir + "/" + opts.save_file + "_" + to_string(epoch) + ".model");
            boost::archive::text_oarchive model_oa(model_out);
            model_oa << model << *encdec;
            model_out.close();
            // preparation for next epoch
            epoch++;
            if(epoch >= opts.start_epoch){
                if(epoch > opts.start_epoch){
                    if((epoch - opts.start_epoch) % opts.decay_for_each == 0){
                        learning_rate *= opts.lr_decay;
                    }
                }else if(epoch == opts.start_epoch){
                    learning_rate *= opts.lr_decay;
                }
            }
            trainer->eta = learning_rate;
            if(opts.guided_alignment == true){
                if(epoch > opts.guided_alignment_start_epoch){
                    if((epoch - opts.guided_alignment_start_epoch) % opts.guided_alignment_decay_for_each == 0){
                        align_w *= opts.guided_alignment_decay;
                    }
                }else if(epoch == opts.guided_alignment_start_epoch){
                    align_w *= opts.guided_alignment_decay;
                }
            }
        }
    }

    void predict(const s2s_options &opts){
        s2s::dicts dicts;
        dicts.load(opts);
        // load model
        dynet::Model model;
        encoder_decoder* encdec = new encoder_decoder(model, &opts);
        //encdec->disable_dropout();
        ifstream model_in(opts.modelfile);
        boost::archive::text_iarchive model_ia(model_in);
        model_ia >> model >> *encdec;
        model_in.close();
        // predict
        s2s::monoling_corpus mono_corp;
        mono_corp.load_src(opts.srcfile, dicts);
        batch one_batch;
        ofstream predict_sents(opts.trgfile);
        encdec->disable_dropout();
        while(mono_corp.next_batch_mono(one_batch, opts.max_batch_pred, dicts)){
            std::vector<std::vector<unsigned int> > osent;
            s2s::greedy_decode(one_batch, osent, encdec, dicts, opts);
            predict_sents << s2s::print_sents(osent, dicts);
        }
        predict_sents.close();
    }

};

int main(int argc, char** argv) {
    namespace po = boost::program_options;
    po::options_description bpo("h");
    s2s::s2s_options opts;
    s2s::set_s2s_options(&bpo, &opts);
    po::variables_map vm;
    po::store(po::parse_command_line(argc, argv, bpo), vm);
    po::notify(vm);
    if(vm.at("mode").as<std::string>() == "train"){
        s2s::add_s2s_options_train(&vm, &opts);
        s2s::check_s2s_options_train(&vm, opts);
        std::string file_name = opts.rootdir + "/options.txt";
        struct stat st;
        if(stat(opts.rootdir.c_str(), &st) != 0){
            mkdir(opts.rootdir.c_str(), 0775);
        }
        ofstream out(file_name);
        boost::archive::text_oarchive oa(out);
        oa << opts;
        out.close();
        dynet::initialize(argc, argv);
        s2s::train(opts);
    }else if(vm.at("mode").as<std::string>() == "predict"){
        ifstream in(opts.rootdir + "/options.txt");
        boost::archive::text_iarchive ia(in);
        ia >> opts;
        in.close();
        s2s::check_s2s_options_predict(&vm, opts);
        dynet::initialize(argc, argv);
        s2s::predict(opts);
    }else if(vm.at("mode").as<std::string>() == "test"){

    }else{
        std::cerr << "Mode does not exist !"<< std::endl;
        assert(false);
    }
}
<commit_msg>update changed<commit_after>#include "dynet/nodes.h"
#include "dynet/dynet.h"
#include "dynet/training.h"
#include "dynet/timing.h"
#include "dynet/rnn.h"
#include "dynet/gru.h"
#include "dynet/lstm.h"
#include "dynet/fast-lstm.h"
#include "dynet/dict.h"
#include "dynet/expr.h"

#include <sys/stat.h>
#include <sys/types.h>
#include <iostream>
#include <fstream>
#include <sstream>
#include <type_traits>
#include <time.h>

#include <boost/serialization/vector.hpp>
#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <boost/program_options.hpp>

#include "s2s/encdec.hpp"
#include "s2s/decode.hpp"
#include "s2s/define.hpp"
#include "s2s/comp.hpp"
#include "s2s/metrics.hpp"
#include "s2s/options.hpp"

namespace s2s {

    void train(const s2s_options &opts){
        s2s::dicts dicts;
        s2s::parallel_corpus para_corp_train;
        s2s::parallel_corpus para_corp_dev;
        dicts.set(opts);
        para_corp_train.load_src(opts.srcfile, dicts);
        para_corp_train.load_trg(opts.trgfile, dicts);
        para_corp_train.load_check();
        para_corp_dev.load_src(opts.srcvalfile, dicts);
        para_corp_dev.load_trg(opts.trgvalfile, dicts);
        para_corp_dev.load_check();
        if(opts.guided_alignment == true){
            para_corp_train.load_align(opts.alignfile);
            para_corp_train.load_check_with_align();
            para_corp_dev.load_align(opts.alignvalfile);
            para_corp_dev.load_check_with_align();
        }
        // for debug
        dicts.save(opts);
        // for debug
        dynet::Model model;
        encoder_decoder* encdec = new encoder_decoder(model, &opts);
        encdec->enable_dropout();
        dynet::Trainer* trainer = nullptr;
        if(opts.optim == "sgd"){
            trainer = new dynet::SimpleSGDTrainer(model);
        }else if(opts.optim == "momentum_sgd"){
            trainer = new dynet::MomentumSGDTrainer(model);
        }else if(opts.optim == "adagrad"){
            trainer = new dynet::AdagradTrainer(model);
        }else if(opts.optim == "adadelta"){
            trainer = new dynet::AdadeltaTrainer(model);
        }else if(opts.optim == "adam"){
            trainer = new dynet::AdamTrainer(model);
        }else{
            std::cerr << "Trainer does not exist !"<< std::endl;
            assert(false);
        }
        float learning_rate = opts.learning_rate;
        trainer->eta0 = learning_rate;
        trainer->eta = learning_rate;
        trainer->eta_decay = 0.f;
        trainer->clip_threshold = opts.clip_threshold;
        trainer->clipping_enabled = opts.clipping_enabled;
        unsigned int epoch = 0;
        float align_w = opts.guided_alignment_weight;
        while(epoch < opts.epochs){
            // train
            para_corp_train.shuffle();
            batch one_batch;
            unsigned int bid = 0;
            while(para_corp_train.next_batch_para(one_batch, opts.max_batch_train, dicts)){
                bid++;
                one_batch.drop_word(dicts, opts);
                auto chrono_start = std::chrono::system_clock::now();
                dynet::ComputationGraph cg;
                std::vector<dynet::expr::Expression> errs_att;
                std::vector<dynet::expr::Expression> errs_out;
                float loss_att = 0.0;
                float loss_out = 0.0;
                std::vector<dynet::expr::Expression> i_enc = encdec->encoder(one_batch, cg);
                std::vector<dynet::expr::Expression> i_feed = encdec->init_feed(one_batch, cg);
                for (unsigned int t = 0; t < one_batch.trg.size() - 1; ++t) {
                    dynet::expr::Expression i_att_t = encdec->decoder_attention(cg, one_batch.trg[t], i_feed[t], i_enc[0]);
                    if(opts.guided_alignment == true){
                        for(unsigned int i = 0; i < one_batch.align.at(t).size(); i++){
                            assert(0 <= one_batch.align.at(t+1).at(i) < one_batch.src.size());
                        }
                        dynet::expr::Expression i_err = pickneglogsoftmax(i_att_t, one_batch.align.at(t+1));
                        errs_att.push_back(i_err);
                    }
                    std::vector<dynet::expr::Expression> i_out_t = encdec->decoder_output(cg, i_att_t, i_enc[1]);
                    i_feed.push_back(i_out_t[1]);
                    dynet::expr::Expression i_err = pickneglogsoftmax(i_out_t[0], one_batch.trg[t+1]);
                    errs_out.push_back(i_err);
                }
                dynet::expr::Expression i_nerr_out = sum_batches(sum(errs_out));
                loss_out = as_scalar(cg.forward(i_nerr_out));
                dynet::expr::Expression i_nerr_all;
                if(opts.guided_alignment == true){
                    dynet::expr::Expression i_nerr_att = sum_batches(sum(errs_att));
                    loss_att = as_scalar(cg.incremental_forward(i_nerr_att));
                    i_nerr_all = i_nerr_out + align_w * i_nerr_att;
                }else{
                    i_nerr_all = i_nerr_out;
                }
                cg.incremental_forward(i_nerr_all);
                cg.backward(i_nerr_all);
                //cg.print_graphviz();
                trainer->update();
                auto chrono_end = std::chrono::system_clock::now();
                auto time_used = (double)std::chrono::duration_cast<std::chrono::milliseconds>(chrono_end - chrono_start).count() / (double)1000;
                std::cerr << "batch: " << bid;
                std::cerr << ",\tsize: " << one_batch.src.at(0).at(0).size();
                std::cerr << ",\toutput loss: " << loss_out;
                std::cerr << ",\tattention loss: " << loss_att;
                std::cerr << ",\tsource length: " << one_batch.src.size();
                std::cerr << ",\ttarget length: " << one_batch.trg.size();
                std::cerr << ",\ttime: " << time_used << " [s]" << std::endl;
                std::cerr << "[epoch=" << trainer->epoch << " eta=" << trainer->eta << " align_w=" << align_w << " clips=" << trainer->clips_since_status << " updates=" << trainer->updates_since_status << "] " << std::endl;
            }
            para_corp_train.reset_index();
            trainer->update_epoch();
            trainer->status();
            std::cerr << std::endl;
            // dev
            encdec->disable_dropout();
            std::cerr << "dev" << std::endl;
            ofstream dev_sents(opts.rootdir + "/dev_" + to_string(epoch) + ".txt");
            while(para_corp_dev.next_batch_para(one_batch, opts.max_batch_pred, dicts)){
                std::vector<std::vector<unsigned int> > osent;
                s2s::greedy_decode(one_batch, osent, encdec, dicts, opts);
                dev_sents << s2s::print_sents(osent, dicts);
            }
            dev_sents.close();
            para_corp_dev.reset_index();
            encdec->enable_dropout();
            // save Model
            ofstream model_out(opts.rootdir + "/" + opts.save_file + "_" + to_string(epoch) + ".model");
            boost::archive::text_oarchive model_oa(model_out);
            model_oa << model << *encdec;
            model_out.close();
            // preparation for next epoch
            epoch++;
            if(epoch >= opts.start_epoch){
                if(epoch > opts.start_epoch){
                    if((epoch - opts.start_epoch) % opts.decay_for_each == 0){
                        learning_rate *= opts.lr_decay;
                    }
                }else if(epoch == opts.start_epoch){
                    learning_rate *= opts.lr_decay;
                }
            }
            trainer->eta = learning_rate;
            if(opts.guided_alignment == true){
                if(epoch > opts.guided_alignment_start_epoch){
                    if((epoch - opts.guided_alignment_start_epoch) % opts.guided_alignment_decay_for_each == 0){
                        align_w *= opts.guided_alignment_decay;
                    }
                }else if(epoch == opts.guided_alignment_start_epoch){
                    align_w *= opts.guided_alignment_decay;
                }
            }
        }
    }

    void predict(const s2s_options &opts){
        s2s::dicts dicts;
        dicts.load(opts);
        // load model
        dynet::Model model;
        encoder_decoder* encdec = new encoder_decoder(model, &opts);
        //encdec->disable_dropout();
        ifstream model_in(opts.modelfile);
        boost::archive::text_iarchive model_ia(model_in);
        model_ia >> model >> *encdec;
        model_in.close();
        // predict
        s2s::monoling_corpus mono_corp;
        mono_corp.load_src(opts.srcfile, dicts);
        batch one_batch;
        ofstream predict_sents(opts.trgfile);
        encdec->disable_dropout();
        while(mono_corp.next_batch_mono(one_batch, opts.max_batch_pred, dicts)){
            std::vector<std::vector<unsigned int> > osent;
            s2s::greedy_decode(one_batch, osent, encdec, dicts, opts);
            predict_sents << s2s::print_sents(osent, dicts);
        }
        predict_sents.close();
    }

};

int main(int argc, char** argv) {
    namespace po = boost::program_options;
    po::options_description bpo("h");
    s2s::s2s_options opts;
    s2s::set_s2s_options(&bpo, &opts);
    po::variables_map vm;
    po::store(po::parse_command_line(argc, argv, bpo), vm);
    po::notify(vm);
    if(vm.at("mode").as<std::string>() == "train"){
        s2s::add_s2s_options_train(&vm, &opts);
        s2s::check_s2s_options_train(&vm, opts);
        std::string file_name = opts.rootdir + "/options.txt";
        struct stat st;
        if(stat(opts.rootdir.c_str(), &st) != 0){
            mkdir(opts.rootdir.c_str(), 0775);
        }
        ofstream out(file_name);
        boost::archive::text_oarchive oa(out);
        oa << opts;
        out.close();
        dynet::initialize(argc, argv);
        s2s::train(opts);
    }else if(vm.at("mode").as<std::string>() == "predict"){
        ifstream in(opts.rootdir + "/options.txt");
        boost::archive::text_iarchive ia(in);
        ia >> opts;
        in.close();
        s2s::check_s2s_options_predict(&vm, opts);
        dynet::initialize(argc, argv);
        s2s::predict(opts);
    }else if(vm.at("mode").as<std::string>() == "test"){

    }else{
        std::cerr << "Mode does not exist !"<< std::endl;
        assert(false);
    }
}
<|endoftext|>


<commit_before>/*
 * ranges_test.cpp
 *
 *  Created on: Dec 16, 2015
 *      Author: zmij
 */

#include <gtest/gtest.h>
#include <awm/util/range.hpp>
#include <sstream>

namespace awm {
namespace util {
namespace test {

TEST(Range, ShortRange)
{
	typedef range<uint16_t> uint16_range;

	uint16_range r1 { 4444, 4744, range_ends::inclusive };
	std::ostringstream os;
	os << r1;
	EXPECT_EQ("[4444,4744]", os.str());
	std::istringstream is("[4444,4744]");
	uint16_range r2;
	bool read_from_stream = is >> r2;
	EXPECT_TRUE(read_from_stream);
	EXPECT_EQ(r1, r2);
}

}  // namespace test
}  // namespace util
}  // namespace awm
<commit_msg>clang: convert istream to bool<commit_after>/*
 * ranges_test.cpp
 *
 *  Created on: Dec 16, 2015
 *      Author: zmij
 */

#include <gtest/gtest.h>
#include <awm/util/range.hpp>
#include <sstream>

namespace awm {
namespace util {
namespace test {

TEST(Range, ShortRange)
{
	typedef range<uint16_t> uint16_range;

	uint16_range r1 { 4444, 4744, range_ends::inclusive };
	std::ostringstream os;
	os << r1;
	EXPECT_EQ("[4444,4744]", os.str());
	std::istringstream is("[4444,4744]");
	uint16_range r2;
	bool read_from_stream = (bool)(is >> r2);
	EXPECT_TRUE(read_from_stream);
	EXPECT_EQ(r1, r2);
}

}  // namespace test
}  // namespace util
}  // namespace awm
<|endoftext|>