<commit_before>//
// Aspia Project
// Copyright (C) 2018 Dmitry Chapyshev <dmitry@aspia.ru>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//

#include "client/file_transfer.h"

#include <QTimerEvent>

#include "base/logging.h"
#include "client/file_status.h"
#include "client/file_transfer_queue_builder.h"
#include "common/file_packet.h"

namespace client {

FileTransfer::FileTransfer(Type type, QObject* parent)
    : QObject(parent),
      type_(type)
{
    actions_.insert(OtherError, QPair<Actions, Action>(Abort, Ask));
    actions_.insert(DirectoryCreateError,
                    QPair<Actions, Action>(Abort | Skip | SkipAll, Ask));
    actions_.insert(FileCreateError,
                    QPair<Actions, Action>(Abort | Skip | SkipAll, Ask));
    actions_.insert(FileOpenError,
                    QPair<Actions, Action>(Abort | Skip | SkipAll, Ask));
    actions_.insert(FileAlreadyExists,
                    QPair<Actions, Action>(Abort | Skip | SkipAll | Replace | ReplaceAll, Ask));
    actions_.insert(FileWriteError,
                    QPair<Actions, Action>(Abort | Skip | SkipAll, Ask));
    actions_.insert(FileReadError,
                    QPair<Actions, Action>(Abort | Skip | SkipAll, Ask));
}

void FileTransfer::start(const QString& source_path,
                         const QString& target_path,
                         const QList<Item>& items)
{
    builder_ = new FileTransferQueueBuilder();

    connect(builder_, &FileTransferQueueBuilder::started, this, &FileTransfer::started);
    connect(builder_, &FileTransferQueueBuilder::error, this, &FileTransfer::taskQueueError);
    connect(builder_, &FileTransferQueueBuilder::finished, this, &FileTransfer::taskQueueReady);

    if (type_ == Downloader)
    {
        connect(builder_, &FileTransferQueueBuilder::newRequest, this, &FileTransfer::remoteRequest);
    }
    else
    {
        DCHECK(type_ == Uploader);
        connect(builder_, &FileTransferQueueBuilder::newRequest, this, &FileTransfer::localRequest);
    }

    connect(builder_, &FileTransferQueueBuilder::finished,
            builder_, &FileTransferQueueBuilder::deleteLater);

    builder_->start(source_path, target_path, items);
}

void FileTransfer::stop()
{
    cancel_timer_id_ = startTimer(std::chrono::seconds(5));
    is_canceled_ = true;
}

FileTransfer::Actions FileTransfer::availableActions(Error error_type) const
{
    return actions_[error_type].first;
}

FileTransfer::Action FileTransfer::defaultAction(Error error_type) const
{
    return actions_[error_type].second;
}

void FileTransfer::setDefaultAction(Error error_type, Action action)
{
    actions_[error_type].second = action;
}

FileTransferTask& FileTransfer::currentTask()
{
    return tasks_.front();
}

void FileTransfer::targetReply(const proto::file_transfer::Request& request,
                               const proto::file_transfer::Reply& reply)
{
    if (tasks_.isEmpty())
        return;

    if (request.has_create_directory_request())
    {
        if (reply.status() == proto::file_transfer::STATUS_SUCCESS ||
            reply.status() == proto::file_transfer::STATUS_PATH_ALREADY_EXISTS)
        {
            processNextTask();
            return;
        }

        processError(DirectoryCreateError,
                     tr("Failed to create directory \"%1\": %2")
                     .arg(currentTask().targetPath())
                     .arg(fileStatusToString(reply.status())));
    }
    else if (request.has_upload_request())
    {
        if (reply.status() != proto::file_transfer::STATUS_SUCCESS)
        {
            Error error_type = FileCreateError;

            if (reply.status() == proto::file_transfer::STATUS_PATH_ALREADY_EXISTS)
                error_type = FileAlreadyExists;

            processError(error_type,
                         tr("Failed to create file \"%1\": %2")
                         .arg(currentTask().targetPath())
                         .arg(fileStatusToString(reply.status())));
            return;
        }

        common::FileRequest* file_request =
            common::FileRequest::packetRequest(proto::file_transfer::PacketRequest::NO_FLAGS);
        connect(file_request, &common::FileRequest::replyReady, this, &FileTransfer::sourceReply);
        sourceRequest(file_request);
    }
    else if (request.has_packet())
    {
        if (reply.status() != proto::file_transfer::STATUS_SUCCESS)
        {
            processError(FileWriteError,
                         tr("Failed to write file \"%1\": %2")
                         .arg(currentTask().targetPath())
                         .arg(fileStatusToString(reply.status())));
            return;
        }

        int64_t full_task_size = currentTask().size();
        if (full_task_size && total_size_)
        {
            int64_t packet_size = common::kMaxFilePacketSize;

            task_transfered_size_ += packet_size;

            if (task_transfered_size_ > full_task_size)
            {
                packet_size = task_transfered_size_ - full_task_size;
                task_transfered_size_ = full_task_size;
            }

            total_transfered_size_ += packet_size;

            int task_percentage = task_transfered_size_ * 100 / full_task_size;
            int total_percentage = total_transfered_size_ * 100 / total_size_;

            if (task_percentage != task_percentage_ || total_percentage != total_percentage_)
            {
                task_percentage_ = task_percentage;
                total_percentage_ = total_percentage;

                emit progressChanged(total_percentage_, task_percentage_);
            }
        }

        if (request.packet().flags() & proto::file_transfer::Packet::LAST_PACKET)
        {
            processNextTask();
            return;
        }

        uint32_t flags = proto::file_transfer::PacketRequest::NO_FLAGS;
        if (is_canceled_)
            flags = proto::file_transfer::PacketRequest::CANCEL;

        common::FileRequest* file_request = common::FileRequest::packetRequest(flags);
        connect(file_request, &common::FileRequest::replyReady, this, &FileTransfer::sourceReply);
        sourceRequest(file_request);
    }
    else
    {
        emit error(this, OtherError, tr("An unexpected response to the request was received"));
    }
}

void FileTransfer::sourceReply(const proto::file_transfer::Request& request,
                               const proto::file_transfer::Reply& reply)
{
    if (tasks_.isEmpty())
        return;

    if (request.has_download_request())
    {
        if (reply.status() != proto::file_transfer::STATUS_SUCCESS)
        {
            processError(FileOpenError,
                         tr("Failed to open file \"%1\": %2")
                         .arg(currentTask().sourcePath()
                         .arg(fileStatusToString(reply.status()))));
            return;
        }

        common::FileRequest* file_request =
            common::FileRequest::uploadRequest(currentTask().targetPath(),
                                               currentTask().overwrite());
        connect(file_request, &common::FileRequest::replyReady, this, &FileTransfer::targetReply);
        targetRequest(file_request);
    }
    else if (request.has_packet_request())
    {
        if (reply.status() != proto::file_transfer::STATUS_SUCCESS)
        {
            processError(FileReadError,
                         tr("Failed to read file \"%1\": %2")
                         .arg(currentTask().sourcePath())
                         .arg(fileStatusToString(reply.status())));
            return;
        }

        common::FileRequest* file_request = common::FileRequest::packet(reply.packet());
        connect(file_request, &common::FileRequest::replyReady, this, &FileTransfer::targetReply);
        targetRequest(file_request);
    }
    else
    {
        emit error(this, OtherError, tr("An unexpected response to the request was received"));
    }
}

void FileTransfer::timerEvent(QTimerEvent* event)
{
    if (event->timerId() == cancel_timer_id_)
    {
        DCHECK(is_canceled_);

        killTimer(cancel_timer_id_);
        cancel_timer_id_ = 0;

        tasks_.clear();

        emit finished();
    }
}

void FileTransfer::taskQueueError(const QString& message)
{
    emit error(this, OtherError, message);
}

void FileTransfer::taskQueueReady()
{
    DCHECK(builder_ != nullptr);

    tasks_ = builder_->taskQueue();

    for (const auto& task : tasks_)
        total_size_ += task.size();

    processTask(false);
}

void FileTransfer::applyAction(Error error_type, Action action)
{
    switch (action)
    {
        case Action::Abort:
            emit finished();
            break;

        case Action::Replace:
        case Action::ReplaceAll:
        {
            if (action == Action::ReplaceAll)
                setDefaultAction(error_type, action);

            processTask(true);
        }
        break;

        case Action::Skip:
        case Action::SkipAll:
        {
            if (action == Action::SkipAll)
                setDefaultAction(error_type, action);

            processNextTask();
        }
        break;

        default:
            LOG(LS_FATAL) << "Unexpected action";
            break;
    }
}

void FileTransfer::processTask(bool overwrite)
{
    task_percentage_ = 0;
    task_transfered_size_ = 0;

    FileTransferTask& task = currentTask();

    task.setOverwrite(overwrite);

    emit currentItemChanged(task.sourcePath(), task.targetPath());

    if (task.isDirectory())
    {
        common::FileRequest* request =
            common::FileRequest::createDirectoryRequest(task.targetPath());
        connect(request, &common::FileRequest::replyReady, this, &FileTransfer::targetReply);
        targetRequest(request);
    }
    else
    {
        common::FileRequest* request =
            common::FileRequest::FileRequest::downloadRequest(task.sourcePath());
        connect(request, &common::FileRequest::replyReady, this, &FileTransfer::sourceReply);
        sourceRequest(request);
    }
}

void FileTransfer::processNextTask()
{
    if (is_canceled_)
        tasks_.clear();

    if (!tasks_.isEmpty())
    {
        // Delete the task only after confirmation of its successful execution.
        tasks_.pop_front();
    }

    if (tasks_.isEmpty())
    {
        if (cancel_timer_id_)
            killTimer(cancel_timer_id_);

        emit finished();
        return;
    }

    processTask(false);
}

void FileTransfer::processError(Error error_type, const QString& message)
{
    Action action = defaultAction(error_type);
    if (action != Ask)
    {
        applyAction(error_type, action);
        return;
    }

    emit error(this, error_type, message);
}

void FileTransfer::sourceRequest(common::FileRequest* request)
{
    if (type_ == Downloader)
    {
        emit remoteRequest(request);
    }
    else
    {
        DCHECK(type_ == Uploader);
        emit localRequest(request);
    }
}

void FileTransfer::targetRequest(common::FileRequest* request)
{
    if (type_ == Downloader)
    {
        emit localRequest(request);
    }
    else
    {
        DCHECK(type_ == Uploader);
        emit remoteRequest(request);
    }
}

} // namespace client
<commit_msg>- Fixed crash when it was impossible to get a list of files to transfer.<commit_after>//
// Aspia Project
// Copyright (C) 2018 Dmitry Chapyshev <dmitry@aspia.ru>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//

#include "client/file_transfer.h"

#include <QTimerEvent>

#include "base/logging.h"
#include "client/file_status.h"
#include "client/file_transfer_queue_builder.h"
#include "common/file_packet.h"

namespace client {

FileTransfer::FileTransfer(Type type, QObject* parent)
    : QObject(parent),
      type_(type)
{
    actions_.insert(OtherError, QPair<Actions, Action>(Abort, Ask));
    actions_.insert(DirectoryCreateError,
                    QPair<Actions, Action>(Abort | Skip | SkipAll, Ask));
    actions_.insert(FileCreateError,
                    QPair<Actions, Action>(Abort | Skip | SkipAll, Ask));
    actions_.insert(FileOpenError,
                    QPair<Actions, Action>(Abort | Skip | SkipAll, Ask));
    actions_.insert(FileAlreadyExists,
                    QPair<Actions, Action>(Abort | Skip | SkipAll | Replace | ReplaceAll, Ask));
    actions_.insert(FileWriteError,
                    QPair<Actions, Action>(Abort | Skip | SkipAll, Ask));
    actions_.insert(FileReadError,
                    QPair<Actions, Action>(Abort | Skip | SkipAll, Ask));
}

void FileTransfer::start(const QString& source_path,
                         const QString& target_path,
                         const QList<Item>& items)
{
    builder_ = new FileTransferQueueBuilder();

    connect(builder_, &FileTransferQueueBuilder::started, this, &FileTransfer::started);
    connect(builder_, &FileTransferQueueBuilder::error, this, &FileTransfer::taskQueueError);
    connect(builder_, &FileTransferQueueBuilder::finished, this, &FileTransfer::taskQueueReady);

    if (type_ == Downloader)
    {
        connect(builder_, &FileTransferQueueBuilder::newRequest, this, &FileTransfer::remoteRequest);
    }
    else
    {
        DCHECK(type_ == Uploader);
        connect(builder_, &FileTransferQueueBuilder::newRequest, this, &FileTransfer::localRequest);
    }

    connect(builder_, &FileTransferQueueBuilder::finished,
            builder_, &FileTransferQueueBuilder::deleteLater);

    builder_->start(source_path, target_path, items);
}

void FileTransfer::stop()
{
    cancel_timer_id_ = startTimer(std::chrono::seconds(5));
    is_canceled_ = true;
}

FileTransfer::Actions FileTransfer::availableActions(Error error_type) const
{
    return actions_[error_type].first;
}

FileTransfer::Action FileTransfer::defaultAction(Error error_type) const
{
    return actions_[error_type].second;
}

void FileTransfer::setDefaultAction(Error error_type, Action action)
{
    actions_[error_type].second = action;
}

FileTransferTask& FileTransfer::currentTask()
{
    return tasks_.front();
}

void FileTransfer::targetReply(const proto::file_transfer::Request& request,
                               const proto::file_transfer::Reply& reply)
{
    if (tasks_.isEmpty())
        return;

    if (request.has_create_directory_request())
    {
        if (reply.status() == proto::file_transfer::STATUS_SUCCESS ||
            reply.status() == proto::file_transfer::STATUS_PATH_ALREADY_EXISTS)
        {
            processNextTask();
            return;
        }

        processError(DirectoryCreateError,
                     tr("Failed to create directory \"%1\": %2")
                     .arg(currentTask().targetPath())
                     .arg(fileStatusToString(reply.status())));
    }
    else if (request.has_upload_request())
    {
        if (reply.status() != proto::file_transfer::STATUS_SUCCESS)
        {
            Error error_type = FileCreateError;

            if (reply.status() == proto::file_transfer::STATUS_PATH_ALREADY_EXISTS)
                error_type = FileAlreadyExists;

            processError(error_type,
                         tr("Failed to create file \"%1\": %2")
                         .arg(currentTask().targetPath())
                         .arg(fileStatusToString(reply.status())));
            return;
        }

        common::FileRequest* file_request =
            common::FileRequest::packetRequest(proto::file_transfer::PacketRequest::NO_FLAGS);
        connect(file_request, &common::FileRequest::replyReady, this, &FileTransfer::sourceReply);
        sourceRequest(file_request);
    }
    else if (request.has_packet())
    {
        if (reply.status() != proto::file_transfer::STATUS_SUCCESS)
        {
            processError(FileWriteError,
                         tr("Failed to write file \"%1\": %2")
                         .arg(currentTask().targetPath())
                         .arg(fileStatusToString(reply.status())));
            return;
        }

        int64_t full_task_size = currentTask().size();
        if (full_task_size && total_size_)
        {
            int64_t packet_size = common::kMaxFilePacketSize;

            task_transfered_size_ += packet_size;

            if (task_transfered_size_ > full_task_size)
            {
                packet_size = task_transfered_size_ - full_task_size;
                task_transfered_size_ = full_task_size;
            }

            total_transfered_size_ += packet_size;

            int task_percentage = task_transfered_size_ * 100 / full_task_size;
            int total_percentage = total_transfered_size_ * 100 / total_size_;

            if (task_percentage != task_percentage_ || total_percentage != total_percentage_)
            {
                task_percentage_ = task_percentage;
                total_percentage_ = total_percentage;

                emit progressChanged(total_percentage_, task_percentage_);
            }
        }

        if (request.packet().flags() & proto::file_transfer::Packet::LAST_PACKET)
        {
            processNextTask();
            return;
        }

        uint32_t flags = proto::file_transfer::PacketRequest::NO_FLAGS;
        if (is_canceled_)
            flags = proto::file_transfer::PacketRequest::CANCEL;

        common::FileRequest* file_request = common::FileRequest::packetRequest(flags);
        connect(file_request, &common::FileRequest::replyReady, this, &FileTransfer::sourceReply);
        sourceRequest(file_request);
    }
    else
    {
        emit error(this, OtherError, tr("An unexpected response to the request was received"));
    }
}

void FileTransfer::sourceReply(const proto::file_transfer::Request& request,
                               const proto::file_transfer::Reply& reply)
{
    if (tasks_.isEmpty())
        return;

    if (request.has_download_request())
    {
        if (reply.status() != proto::file_transfer::STATUS_SUCCESS)
        {
            processError(FileOpenError,
                         tr("Failed to open file \"%1\": %2")
                         .arg(currentTask().sourcePath()
                         .arg(fileStatusToString(reply.status()))));
            return;
        }

        common::FileRequest* file_request =
            common::FileRequest::uploadRequest(currentTask().targetPath(),
                                               currentTask().overwrite());
        connect(file_request, &common::FileRequest::replyReady, this, &FileTransfer::targetReply);
        targetRequest(file_request);
    }
    else if (request.has_packet_request())
    {
        if (reply.status() != proto::file_transfer::STATUS_SUCCESS)
        {
            processError(FileReadError,
                         tr("Failed to read file \"%1\": %2")
                         .arg(currentTask().sourcePath())
                         .arg(fileStatusToString(reply.status())));
            return;
        }

        common::FileRequest* file_request = common::FileRequest::packet(reply.packet());
        connect(file_request, &common::FileRequest::replyReady, this, &FileTransfer::targetReply);
        targetRequest(file_request);
    }
    else
    {
        emit error(this, OtherError, tr("An unexpected response to the request was received"));
    }
}

void FileTransfer::timerEvent(QTimerEvent* event)
{
    if (event->timerId() == cancel_timer_id_)
    {
        DCHECK(is_canceled_);

        killTimer(cancel_timer_id_);
        cancel_timer_id_ = 0;

        tasks_.clear();

        emit finished();
    }
}

void FileTransfer::taskQueueError(const QString& message)
{
    emit error(this, OtherError, message);
}

void FileTransfer::taskQueueReady()
{
    DCHECK(builder_ != nullptr);

    tasks_ = builder_->taskQueue();
    if (tasks_.isEmpty())
    {
        emit finished();
        return;
    }

    for (const auto& task : tasks_)
        total_size_ += task.size();

    processTask(false);
}

void FileTransfer::applyAction(Error error_type, Action action)
{
    switch (action)
    {
        case Action::Abort:
            emit finished();
            break;

        case Action::Replace:
        case Action::ReplaceAll:
        {
            if (action == Action::ReplaceAll)
                setDefaultAction(error_type, action);

            processTask(true);
        }
        break;

        case Action::Skip:
        case Action::SkipAll:
        {
            if (action == Action::SkipAll)
                setDefaultAction(error_type, action);

            processNextTask();
        }
        break;

        default:
            LOG(LS_FATAL) << "Unexpected action";
            break;
    }
}

void FileTransfer::processTask(bool overwrite)
{
    task_percentage_ = 0;
    task_transfered_size_ = 0;

    FileTransferTask& task = currentTask();

    task.setOverwrite(overwrite);

    emit currentItemChanged(task.sourcePath(), task.targetPath());

    if (task.isDirectory())
    {
        common::FileRequest* request =
            common::FileRequest::createDirectoryRequest(task.targetPath());
        connect(request, &common::FileRequest::replyReady, this, &FileTransfer::targetReply);
        targetRequest(request);
    }
    else
    {
        common::FileRequest* request =
            common::FileRequest::FileRequest::downloadRequest(task.sourcePath());
        connect(request, &common::FileRequest::replyReady, this, &FileTransfer::sourceReply);
        sourceRequest(request);
    }
}

void FileTransfer::processNextTask()
{
    if (is_canceled_)
        tasks_.clear();

    if (!tasks_.isEmpty())
    {
        // Delete the task only after confirmation of its successful execution.
        tasks_.pop_front();
    }

    if (tasks_.isEmpty())
    {
        if (cancel_timer_id_)
            killTimer(cancel_timer_id_);

        emit finished();
        return;
    }

    processTask(false);
}

void FileTransfer::processError(Error error_type, const QString& message)
{
    Action action = defaultAction(error_type);
    if (action != Ask)
    {
        applyAction(error_type, action);
        return;
    }

    emit error(this, error_type, message);
}

void FileTransfer::sourceRequest(common::FileRequest* request)
{
    if (type_ == Downloader)
    {
        emit remoteRequest(request);
    }
    else
    {
        DCHECK(type_ == Uploader);
        emit localRequest(request);
    }
}

void FileTransfer::targetRequest(common::FileRequest* request)
{
    if (type_ == Downloader)
    {
        emit localRequest(request);
    }
    else
    {
        DCHECK(type_ == Uploader);
        emit remoteRequest(request);
    }
}

} // namespace client
<|endoftext|>


<commit_before>#include <bits/stdc++.h>
using namespace std;

int main() {
    int n, m, rurka;
    static int min[300001];

    scanf("%d %d", &n, &m);

    int ostatni = n+1;
    scanf("%d", &min[0]); // min[0] = r[0];

    //minima
    for(int i=1; i<n; i++)
    {
        scanf("%d", &rurka);
        min[i] = (min[i-1]<rurka ? min[i-1] : rurka);
    }

    for(int j=0; j<m; j++) {

        int a = ostatni-1;
        scanf("%d", &rurka);

        if(rurka > min[0]) { ostatni = 0; break; }

        while(a >= 0 and (rurka > min[a])) a--;
        ostatni = a;

        if(ostatni == 0) break;
    }

    if(ostatni == 0) printf("0\n");
    else printf("%d\n", ostatni+1);

    return 0;
}

<commit_msg>oi/oi15_kra.cpp<commit_after>#include <bits/stdc++.h>
using namespace std;

int main() {
    int n, m, rurka;
    static int min[300001];

    scanf("%d %d", &n, &m);

    int ostatni = n+1;
    scanf("%d", &min[0]); // min[0] = r[0];

    // min array
    for(int i=1; i<n; i++) {
        scanf("%d", &rurka);
        min[i] = (min[i-1]<rurka ? min[i-1] : rurka);
    }

    for(int j=0; j<m; j++) {

        int a = ostatni-1;
        scanf("%d", &rurka);

        if(rurka > min[0]) { ostatni = 0; break; }

        while(a >= 0 and (rurka > min[a])) a--;
        ostatni = a;

        if(ostatni == 0) break;
    }

    if(ostatni == 0) printf("0\n");
    else printf("%d\n", ostatni+1);

    return 0;
}

<|endoftext|>


<commit_before>//--------------------------------------------------------------------*- C++ -*-
// CLING - the C++ LLVM-based InterpreterG :)
// author:  Axel Naumann <axel@cern.ch>
//
// This file is dual-licensed: you can choose to license it under the University
// of Illinois Open Source License or the GNU Lesser General Public License. See
// LICENSE.TXT for details.
//------------------------------------------------------------------------------

#include "IncrementalJIT.h"

#include "IncrementalExecutor.h"

#include "llvm/ExecutionEngine/Orc/LambdaResolver.h"
#include "llvm/Support/DynamicLibrary.h"

#ifdef __APPLE__
// Apple adds an extra '_'
# define MANGLE_PREFIX "_"
#else
# define MANGLE_PREFIX ""
#endif

using namespace llvm;

namespace {
// Forward cxa_atexit for global d'tors.
static void local_cxa_atexit(void (*func) (void*), void* arg, void* dso) {
  cling::IncrementalExecutor* exe = (cling::IncrementalExecutor*)dso;
  exe->AddAtExitFunc(func, arg);
}

///\brief Memory manager providing the lop-level link to the
/// IncrementalExecutor, handles missing or special / replaced symbols.
class ClingMemoryManager: public SectionMemoryManager {
  cling::IncrementalExecutor& m_exe;

public:
  ClingMemoryManager(cling::IncrementalExecutor& Exe):
    m_exe(Exe) {}

  ///\brief Simply wraps the base class's function setting AbortOnFailure
  /// to false and instead using the error handling mechanism to report it.
  void* getPointerToNamedFunction(const std::string &Name,
                                  bool /*AbortOnFailure*/ =true) override {
    return SectionMemoryManager::getPointerToNamedFunction(Name, false);
  }
};

  class NotifyFinalizedT {
  public:
    NotifyFinalizedT(cling::IncrementalJIT &jit) : m_JIT(jit) {}
    void operator()(llvm::orc::ObjectLinkingLayerBase::ObjSetHandleT H) {
      m_JIT.RemoveUnfinalizedSection(H);
    }

  private:
    cling::IncrementalJIT &m_JIT;
  };

} // unnamed namespace

namespace cling {

///\brief Memory manager for the OrcJIT layers to resolve symbols from the
/// common IncrementalJIT. I.e. the master of the Orcs.
/// Each ObjectLayer instance has one Azog object.
class Azog: public RTDyldMemoryManager {
  cling::IncrementalJIT& m_jit;

public:
  Azog(cling::IncrementalJIT& Jit): m_jit(Jit) {}

  RTDyldMemoryManager* getExeMM() const { return m_jit.m_ExeMM.get(); }

  uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment,
                               unsigned SectionID,
                               StringRef SectionName) override {
    uint8_t *Addr =
      getExeMM()->allocateCodeSection(Size, Alignment, SectionID, SectionName);
    m_jit.m_SectionsAllocatedSinceLastLoad.insert(Addr);
    return Addr;
  }

  uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment,
                               unsigned SectionID, StringRef SectionName,
                               bool IsReadOnly) override {
    uint8_t *Addr = getExeMM()->allocateDataSection(Size, Alignment, SectionID,
                                                    SectionName, IsReadOnly);
    m_jit.m_SectionsAllocatedSinceLastLoad.insert(Addr);
    return Addr;
  }

  void reserveAllocationSpace(uintptr_t CodeSize, uint32_t CodeAlign,
                              uintptr_t RODataSize, uint32_t RODataAlign,
                              uintptr_t RWDataSize, uint32_t RWDataAlign) override {
    return getExeMM()->reserveAllocationSpace(CodeSize, CodeAlign, RODataSize,
                                              RODataAlign, RWDataSize,
                                              RWDataAlign);
  }

  bool needsToReserveAllocationSpace() override {
    return getExeMM()->needsToReserveAllocationSpace();
  }

  void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr,
                        size_t Size) override {
    return getExeMM()->registerEHFrames(Addr, LoadAddr, Size);
  }

  void deregisterEHFrames(uint8_t *Addr, uint64_t LoadAddr,
                          size_t Size) override {
    return getExeMM()->deregisterEHFrames(Addr, LoadAddr, Size);
  }

  uint64_t getSymbolAddress(const std::string &Name) override {
    return m_jit.getSymbolAddressWithoutMangling(Name,
                                                 true /*also use dlsym*/)
      .getAddress();
  }

  void *getPointerToNamedFunction(const std::string &Name,
                                  bool AbortOnFailure = true) override {
    return getExeMM()->getPointerToNamedFunction(Name, AbortOnFailure);
  }

  using llvm::RuntimeDyld::MemoryManager::notifyObjectLoaded;

  void notifyObjectLoaded(ExecutionEngine *EE,
                          const object::ObjectFile &O) override {
    return getExeMM()->notifyObjectLoaded(EE, O);
  }

  bool finalizeMemory(std::string *ErrMsg = nullptr) override {
    // Each set of objects loaded will be finalized exactly once, but since
    // symbol lookup during relocation may recursively trigger the
    // loading/relocation of other modules, and since we're forwarding all
    // finalizeMemory calls to a single underlying memory manager, we need to
    // defer forwarding the call on until all necessary objects have been
    // loaded. Otherwise, during the relocation of a leaf object, we will end
    // up finalizing memory, causing a crash further up the stack when we
    // attempt to apply relocations to finalized memory.
    // To avoid finalizing too early, look at how many objects have been
    // loaded but not yet finalized. This is a bit of a hack that relies on
    // the fact that we're lazily emitting object files: The only way you can
    // get more than one set of objects loaded but not yet finalized is if
    // they were loaded during relocation of another set.
    if (m_jit.m_UnfinalizedSections.size() == 1)
      return getExeMM()->finalizeMemory(ErrMsg);
    return false;
  };

}; // class Azog

IncrementalJIT::IncrementalJIT(IncrementalExecutor& exe,
                               std::unique_ptr<TargetMachine> TM):
  m_Parent(exe),
  m_TM(std::move(TM)),
  m_TMDataLayout(m_TM->createDataLayout()),
  m_ExeMM(llvm::make_unique<ClingMemoryManager>(m_Parent)),
  m_NotifyObjectLoaded(*this),
  m_ObjectLayer(m_SymbolMap, m_NotifyObjectLoaded, NotifyFinalizedT(*this)),
  m_CompileLayer(m_ObjectLayer, llvm::orc::SimpleCompiler(*m_TM)),
  m_LazyEmitLayer(m_CompileLayer) {

  // Enable JIT symbol resolution from the binary.
  llvm::sys::DynamicLibrary::LoadLibraryPermanently(0, 0);

  // Make debug symbols available.
  m_GDBListener = 0; // JITEventListener::createGDBRegistrationListener();

// #if MCJIT
//   llvm::EngineBuilder builder(std::move(m));

//   std::string errMsg;
//   builder.setErrorStr(&errMsg);
//   builder.setOptLevel(llvm::CodeGenOpt::Less);
//   builder.setEngineKind(llvm::EngineKind::JIT);
//   std::unique_ptr<llvm::RTDyldMemoryManager>
//     MemMan(new ClingMemoryManager(*this));
//   builder.setMCJITMemoryManager(std::move(MemMan));

//   // EngineBuilder uses default c'ted TargetOptions, too:
//   llvm::TargetOptions TargetOpts;
//   TargetOpts.NoFramePointerElim = 1;
//   TargetOpts.JITEmitDebugInfo = 1;

//   builder.setTargetOptions(TargetOpts);

//   m_engine.reset(builder.create());
//   assert(m_engine && "Cannot create module!");
// #endif
}

llvm::orc::JITSymbol
IncrementalJIT::getInjectedSymbols(llvm::StringRef Name) const {
  using JITSymbol = llvm::orc::JITSymbol;
  if (Name == MANGLE_PREFIX "__cxa_atexit") {
    // Rewire __cxa_atexit to ~Interpreter(), thus also global destruction
    // coming from the JIT.
    return JITSymbol((uint64_t)&local_cxa_atexit,
                     llvm::JITSymbolFlags::Exported);
  } else if (Name == MANGLE_PREFIX "__dso_handle") {
    // Provide IncrementalExecutor as the third argument to __cxa_atexit.
    return JITSymbol((uint64_t)&m_Parent,
                     llvm::JITSymbolFlags::Exported);
  }

  auto SymMapI = m_SymbolMap.find(Name);
  if (SymMapI != m_SymbolMap.end())
    return JITSymbol(SymMapI->second, llvm::JITSymbolFlags::Exported);

  return JITSymbol(nullptr);
}

llvm::orc::JITSymbol
IncrementalJIT::getSymbolAddressWithoutMangling(llvm::StringRef Name,
                                                bool AlsoInProcess) {
  if (auto Sym = getInjectedSymbols(Name))
    return Sym;

  if (AlsoInProcess) {
    if (RuntimeDyld::SymbolInfo SymInfo = m_ExeMM->findSymbol(Name))
      return llvm::orc::JITSymbol(SymInfo.getAddress(),
                                  llvm::JITSymbolFlags::Exported);
  }

  if (auto Sym = m_LazyEmitLayer.findSymbol(Name, false))
    return Sym;

  return llvm::orc::JITSymbol(nullptr);
}

size_t IncrementalJIT::addModules(std::vector<llvm::Module*>&& modules) {
  // If this module doesn't have a DataLayout attached then attach the
  // default.
  for (auto&& mod: modules) {
    mod->setDataLayout(m_TMDataLayout);
  }

  // LLVM MERGE FIXME: update this to use new interfaces.
  auto Resolver = llvm::orc::createLambdaResolver(
    [&](const std::string &S) {
      if (auto Sym = getInjectedSymbols(S))
        return RuntimeDyld::SymbolInfo((uint64_t)Sym.getAddress(),
                                       Sym.getFlags());
      return m_ExeMM->findSymbol(S);
    },
    [&](const std::string &Name) {
      if (auto Sym = getSymbolAddressWithoutMangling(Name, true)
          /*was: findSymbol(Name)*/)
        return RuntimeDyld::SymbolInfo(Sym.getAddress(),
                                       Sym.getFlags());

      /// This method returns the address of the specified function or variable
      /// that could not be resolved by getSymbolAddress() or by resolving
      /// possible weak symbols by the ExecutionEngine.
      /// It is used to resolve symbols during module linking.

      std::string NameNoPrefix;
      if (MANGLE_PREFIX[0]
          && !Name.compare(0, strlen(MANGLE_PREFIX), MANGLE_PREFIX))
        NameNoPrefix = Name.substr(strlen(MANGLE_PREFIX), -1);
      else
        NameNoPrefix = std::move(Name);
      uint64_t addr
        = (uint64_t) getParent().NotifyLazyFunctionCreators(NameNoPrefix);
      return RuntimeDyld::SymbolInfo(addr, llvm::JITSymbolFlags::Weak);
    });

  ModuleSetHandleT MSHandle
    = m_LazyEmitLayer.addModuleSet(std::move(modules),
                                   llvm::make_unique<Azog>(*this),
                                   std::move(Resolver));
  m_UnloadPoints.push_back(MSHandle);
  return m_UnloadPoints.size() - 1;
}

// void* IncrementalJIT::finalizeMemory() {
//   for (auto &P : UnfinalizedSections)
//     if (P.second.count(LocalAddress))
//       ObjectLayer.mapSectionAddress(P.first, LocalAddress, TargetAddress);
// }

void IncrementalJIT::removeModules(size_t handle) {
  if (handle == (size_t)-1)
    return;
  auto objSetHandle = m_UnloadPoints[handle];
  m_LazyEmitLayer.removeModuleSet(objSetHandle);
}

}// end namespace cling
<commit_msg>Remove m_exe variable from ClingMemoryManager as it's not being used.<commit_after>//--------------------------------------------------------------------*- C++ -*-
// CLING - the C++ LLVM-based InterpreterG :)
// author:  Axel Naumann <axel@cern.ch>
//
// This file is dual-licensed: you can choose to license it under the University
// of Illinois Open Source License or the GNU Lesser General Public License. See
// LICENSE.TXT for details.
//------------------------------------------------------------------------------

#include "IncrementalJIT.h"

#include "IncrementalExecutor.h"

#include "llvm/ExecutionEngine/Orc/LambdaResolver.h"
#include "llvm/Support/DynamicLibrary.h"

#ifdef __APPLE__
// Apple adds an extra '_'
# define MANGLE_PREFIX "_"
#else
# define MANGLE_PREFIX ""
#endif

using namespace llvm;

namespace {
// Forward cxa_atexit for global d'tors.
static void local_cxa_atexit(void (*func) (void*), void* arg, void* dso) {
  cling::IncrementalExecutor* exe = (cling::IncrementalExecutor*)dso;
  exe->AddAtExitFunc(func, arg);
}

///\brief Memory manager providing the lop-level link to the
/// IncrementalExecutor, handles missing or special / replaced symbols.
class ClingMemoryManager: public SectionMemoryManager {
public:
  ClingMemoryManager(cling::IncrementalExecutor& Exe) {}

  ///\brief Simply wraps the base class's function setting AbortOnFailure
  /// to false and instead using the error handling mechanism to report it.
  void* getPointerToNamedFunction(const std::string &Name,
                                  bool /*AbortOnFailure*/ =true) override {
    return SectionMemoryManager::getPointerToNamedFunction(Name, false);
  }
};

  class NotifyFinalizedT {
  public:
    NotifyFinalizedT(cling::IncrementalJIT &jit) : m_JIT(jit) {}
    void operator()(llvm::orc::ObjectLinkingLayerBase::ObjSetHandleT H) {
      m_JIT.RemoveUnfinalizedSection(H);
    }

  private:
    cling::IncrementalJIT &m_JIT;
  };

} // unnamed namespace

namespace cling {

///\brief Memory manager for the OrcJIT layers to resolve symbols from the
/// common IncrementalJIT. I.e. the master of the Orcs.
/// Each ObjectLayer instance has one Azog object.
class Azog: public RTDyldMemoryManager {
  cling::IncrementalJIT& m_jit;

public:
  Azog(cling::IncrementalJIT& Jit): m_jit(Jit) {}

  RTDyldMemoryManager* getExeMM() const { return m_jit.m_ExeMM.get(); }

  uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment,
                               unsigned SectionID,
                               StringRef SectionName) override {
    uint8_t *Addr =
      getExeMM()->allocateCodeSection(Size, Alignment, SectionID, SectionName);
    m_jit.m_SectionsAllocatedSinceLastLoad.insert(Addr);
    return Addr;
  }

  uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment,
                               unsigned SectionID, StringRef SectionName,
                               bool IsReadOnly) override {
    uint8_t *Addr = getExeMM()->allocateDataSection(Size, Alignment, SectionID,
                                                    SectionName, IsReadOnly);
    m_jit.m_SectionsAllocatedSinceLastLoad.insert(Addr);
    return Addr;
  }

  void reserveAllocationSpace(uintptr_t CodeSize, uint32_t CodeAlign,
                              uintptr_t RODataSize, uint32_t RODataAlign,
                              uintptr_t RWDataSize, uint32_t RWDataAlign) override {
    return getExeMM()->reserveAllocationSpace(CodeSize, CodeAlign, RODataSize,
                                              RODataAlign, RWDataSize,
                                              RWDataAlign);
  }

  bool needsToReserveAllocationSpace() override {
    return getExeMM()->needsToReserveAllocationSpace();
  }

  void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr,
                        size_t Size) override {
    return getExeMM()->registerEHFrames(Addr, LoadAddr, Size);
  }

  void deregisterEHFrames(uint8_t *Addr, uint64_t LoadAddr,
                          size_t Size) override {
    return getExeMM()->deregisterEHFrames(Addr, LoadAddr, Size);
  }

  uint64_t getSymbolAddress(const std::string &Name) override {
    return m_jit.getSymbolAddressWithoutMangling(Name,
                                                 true /*also use dlsym*/)
      .getAddress();
  }

  void *getPointerToNamedFunction(const std::string &Name,
                                  bool AbortOnFailure = true) override {
    return getExeMM()->getPointerToNamedFunction(Name, AbortOnFailure);
  }

  using llvm::RuntimeDyld::MemoryManager::notifyObjectLoaded;

  void notifyObjectLoaded(ExecutionEngine *EE,
                          const object::ObjectFile &O) override {
    return getExeMM()->notifyObjectLoaded(EE, O);
  }

  bool finalizeMemory(std::string *ErrMsg = nullptr) override {
    // Each set of objects loaded will be finalized exactly once, but since
    // symbol lookup during relocation may recursively trigger the
    // loading/relocation of other modules, and since we're forwarding all
    // finalizeMemory calls to a single underlying memory manager, we need to
    // defer forwarding the call on until all necessary objects have been
    // loaded. Otherwise, during the relocation of a leaf object, we will end
    // up finalizing memory, causing a crash further up the stack when we
    // attempt to apply relocations to finalized memory.
    // To avoid finalizing too early, look at how many objects have been
    // loaded but not yet finalized. This is a bit of a hack that relies on
    // the fact that we're lazily emitting object files: The only way you can
    // get more than one set of objects loaded but not yet finalized is if
    // they were loaded during relocation of another set.
    if (m_jit.m_UnfinalizedSections.size() == 1)
      return getExeMM()->finalizeMemory(ErrMsg);
    return false;
  };

}; // class Azog

IncrementalJIT::IncrementalJIT(IncrementalExecutor& exe,
                               std::unique_ptr<TargetMachine> TM):
  m_Parent(exe),
  m_TM(std::move(TM)),
  m_TMDataLayout(m_TM->createDataLayout()),
  m_ExeMM(llvm::make_unique<ClingMemoryManager>(m_Parent)),
  m_NotifyObjectLoaded(*this),
  m_ObjectLayer(m_SymbolMap, m_NotifyObjectLoaded, NotifyFinalizedT(*this)),
  m_CompileLayer(m_ObjectLayer, llvm::orc::SimpleCompiler(*m_TM)),
  m_LazyEmitLayer(m_CompileLayer) {

  // Enable JIT symbol resolution from the binary.
  llvm::sys::DynamicLibrary::LoadLibraryPermanently(0, 0);

  // Make debug symbols available.
  m_GDBListener = 0; // JITEventListener::createGDBRegistrationListener();

// #if MCJIT
//   llvm::EngineBuilder builder(std::move(m));

//   std::string errMsg;
//   builder.setErrorStr(&errMsg);
//   builder.setOptLevel(llvm::CodeGenOpt::Less);
//   builder.setEngineKind(llvm::EngineKind::JIT);
//   std::unique_ptr<llvm::RTDyldMemoryManager>
//     MemMan(new ClingMemoryManager(*this));
//   builder.setMCJITMemoryManager(std::move(MemMan));

//   // EngineBuilder uses default c'ted TargetOptions, too:
//   llvm::TargetOptions TargetOpts;
//   TargetOpts.NoFramePointerElim = 1;
//   TargetOpts.JITEmitDebugInfo = 1;

//   builder.setTargetOptions(TargetOpts);

//   m_engine.reset(builder.create());
//   assert(m_engine && "Cannot create module!");
// #endif
}

llvm::orc::JITSymbol
IncrementalJIT::getInjectedSymbols(llvm::StringRef Name) const {
  using JITSymbol = llvm::orc::JITSymbol;
  if (Name == MANGLE_PREFIX "__cxa_atexit") {
    // Rewire __cxa_atexit to ~Interpreter(), thus also global destruction
    // coming from the JIT.
    return JITSymbol((uint64_t)&local_cxa_atexit,
                     llvm::JITSymbolFlags::Exported);
  } else if (Name == MANGLE_PREFIX "__dso_handle") {
    // Provide IncrementalExecutor as the third argument to __cxa_atexit.
    return JITSymbol((uint64_t)&m_Parent,
                     llvm::JITSymbolFlags::Exported);
  }

  auto SymMapI = m_SymbolMap.find(Name);
  if (SymMapI != m_SymbolMap.end())
    return JITSymbol(SymMapI->second, llvm::JITSymbolFlags::Exported);

  return JITSymbol(nullptr);
}

llvm::orc::JITSymbol
IncrementalJIT::getSymbolAddressWithoutMangling(llvm::StringRef Name,
                                                bool AlsoInProcess) {
  if (auto Sym = getInjectedSymbols(Name))
    return Sym;

  if (AlsoInProcess) {
    if (RuntimeDyld::SymbolInfo SymInfo = m_ExeMM->findSymbol(Name))
      return llvm::orc::JITSymbol(SymInfo.getAddress(),
                                  llvm::JITSymbolFlags::Exported);
  }

  if (auto Sym = m_LazyEmitLayer.findSymbol(Name, false))
    return Sym;

  return llvm::orc::JITSymbol(nullptr);
}

size_t IncrementalJIT::addModules(std::vector<llvm::Module*>&& modules) {
  // If this module doesn't have a DataLayout attached then attach the
  // default.
  for (auto&& mod: modules) {
    mod->setDataLayout(m_TMDataLayout);
  }

  // LLVM MERGE FIXME: update this to use new interfaces.
  auto Resolver = llvm::orc::createLambdaResolver(
    [&](const std::string &S) {
      if (auto Sym = getInjectedSymbols(S))
        return RuntimeDyld::SymbolInfo((uint64_t)Sym.getAddress(),
                                       Sym.getFlags());
      return m_ExeMM->findSymbol(S);
    },
    [&](const std::string &Name) {
      if (auto Sym = getSymbolAddressWithoutMangling(Name, true)
          /*was: findSymbol(Name)*/)
        return RuntimeDyld::SymbolInfo(Sym.getAddress(),
                                       Sym.getFlags());

      /// This method returns the address of the specified function or variable
      /// that could not be resolved by getSymbolAddress() or by resolving
      /// possible weak symbols by the ExecutionEngine.
      /// It is used to resolve symbols during module linking.

      std::string NameNoPrefix;
      if (MANGLE_PREFIX[0]
          && !Name.compare(0, strlen(MANGLE_PREFIX), MANGLE_PREFIX))
        NameNoPrefix = Name.substr(strlen(MANGLE_PREFIX), -1);
      else
        NameNoPrefix = std::move(Name);
      uint64_t addr
        = (uint64_t) getParent().NotifyLazyFunctionCreators(NameNoPrefix);
      return RuntimeDyld::SymbolInfo(addr, llvm::JITSymbolFlags::Weak);
    });

  ModuleSetHandleT MSHandle
    = m_LazyEmitLayer.addModuleSet(std::move(modules),
                                   llvm::make_unique<Azog>(*this),
                                   std::move(Resolver));
  m_UnloadPoints.push_back(MSHandle);
  return m_UnloadPoints.size() - 1;
}

// void* IncrementalJIT::finalizeMemory() {
//   for (auto &P : UnfinalizedSections)
//     if (P.second.count(LocalAddress))
//       ObjectLayer.mapSectionAddress(P.first, LocalAddress, TargetAddress);
// }

void IncrementalJIT::removeModules(size_t handle) {
  if (handle == (size_t)-1)
    return;
  auto objSetHandle = m_UnloadPoints[handle];
  m_LazyEmitLayer.removeModuleSet(objSetHandle);
}

}// end namespace cling
<|endoftext|>


<commit_before>/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "paddle/fluid/framework/parallel_executor.h"

#include "ThreadPool.h"

#ifdef PADDLE_WITH_CUDA
#include "paddle/fluid/platform/nccl_helper.h"
#endif

#include "paddle/fluid/framework/details/multi_devices_graph_builder.h"
#include "paddle/fluid/framework/details/threaded_ssa_graph_executor.h"

namespace paddle {
namespace framework {

class ParallelExecutorPrivate {
 public:
  explicit ParallelExecutorPrivate(const std::vector<platform::Place> &places)
      : places_(places) {}

  std::vector<platform::Place> places_;
  std::vector<Scope *> local_scopes_;
  Scope *global_scope_;
  std::unique_ptr<details::SSAGraphExecutor> executor_;

#ifdef PADDLE_WITH_CUDA
  std::unique_ptr<platform::NCCLContextMap> nccl_ctxs_;
#endif
};

ParallelExecutor::ParallelExecutor(
    size_t num_threads, bool use_event,
    const std::vector<platform::Place> &places,
    const std::unordered_set<std::string> &params,
    const ProgramDesc &startup_program, const ProgramDesc &main_program,
    const std::string &loss_var_name, Scope *scope)
    : member_(new ParallelExecutorPrivate(places)) {
  member_->global_scope_ = scope;

  // Step 1. RunStartupProgram and Bcast the params to devs.
  Executor exe(places[0]);
  exe.Run(startup_program, scope, 0);
  // Create local scopes
  for (size_t i = 0; i < member_->places_.size(); ++i) {
    member_->local_scopes_.push_back(&scope->NewScope());
  }

// Bcast Parameters to all GPUs
#ifdef PADDLE_WITH_CUDA
  member_->nccl_ctxs_.reset(new platform::NCCLContextMap(member_->places_));
#endif
  if (platform::is_gpu_place(places[0]) &&
      member_->local_scopes_.size() != 1) {  // Is CUDA
    BCastParamsToGPUs(startup_program);
  }
// Startup Program has been run. All local scopes has correct parameters.

// Step 2. Convert main_program to SSA form and dependency graph. Also, insert
// ncclOp
#ifdef PADDLE_WITH_CUDA
  details::MultiDevSSAGraphBuilder builder(member_->places_, loss_var_name,
                                           params, member_->local_scopes_,
                                           member_->nccl_ctxs_.get());
#else
  details::MultiDevSSAGraphBuilder builder(member_->places_, loss_var_name,
                                           params, member_->local_scopes_);
#endif
  auto graph = builder.Build(main_program);

  member_->executor_.reset(new details::ThreadedSSAGraphExecutor(
      num_threads, use_event, member_->local_scopes_, places,
      std::move(graph)));

  // Step 3. Create vars in each scope;
  for (auto *scope : member_->local_scopes_) {
    for (auto *var : main_program.Block(0).AllVars()) {
      if (scope->FindVar(var->Name()) != nullptr) {
        continue;
      }

      InitializeVariable(scope->Var(var->Name()), var->GetType());
    }
  }
}

void ParallelExecutor::BCastParamsToGPUs(
    const ProgramDesc &startup_program) const {
#ifdef PADDLE_WITH_CUDA
  auto *main_scope = member_->local_scopes_[0];

  for (auto *var_desc : startup_program.Block(0).AllVars()) {
    if (var_desc->GetType() == proto::VarType::LOD_TENSOR) {
      auto &main_tensor =
          main_scope->FindVar(var_desc->Name())->Get<LoDTensor>();
      ncclDataType_t data_type = platform::ToNCCLDataType(main_tensor.type());
      auto &dims = main_tensor.dims();
      size_t numel = main_tensor.numel();

      platform::NCCLGroupGuard guard;

      for (size_t i = 0; i < member_->places_.size(); ++i) {
        auto place = member_->places_[i];
        void *buffer;
        if (i == 0) {
          buffer = const_cast<void *>(main_tensor.data<void>());
        } else {
          auto local_scope = member_->local_scopes_[i];
          auto *t = local_scope->Var(var_desc->Name())->GetMutable<LoDTensor>();
          t->Resize(dims);
          buffer = t->mutable_data(place, main_tensor.type());
        }

        auto &nccl_ctx = member_->nccl_ctxs_->at(place);
        platform::dynload::ncclBcast(buffer, numel, data_type, 0,
                                     nccl_ctx.comm_, nccl_ctx.stream());
      }
    }
    member_->nccl_ctxs_->WaitAll();
  }
#else
  PADDLE_THROW("Not compiled with CUDA");
#endif
}

void ParallelExecutor::Run(const std::vector<std::string> &fetch_tensors,
                           const std::string &fetched_var_name) {
  auto fetch_data = member_->executor_->Run(fetch_tensors);
  *member_->global_scope_->Var(fetched_var_name)->GetMutable<FeedFetchList>() =
      fetch_data;
}

}  // namespace framework
}  // namespace paddle
<commit_msg>refine parallel<commit_after>/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "paddle/fluid/framework/parallel_executor.h"
#include <string>

#include "ThreadPool.h"

#ifdef PADDLE_WITH_CUDA
#include "paddle/fluid/platform/nccl_helper.h"
#endif

#include "paddle/fluid/framework/details/multi_devices_graph_builder.h"
#include "paddle/fluid/framework/details/threaded_ssa_graph_executor.h"

namespace paddle {
namespace framework {

class ParallelExecutorPrivate {
 public:
  explicit ParallelExecutorPrivate(const std::vector<platform::Place> &places)
      : places_(places) {}

  std::vector<platform::Place> places_;
  std::vector<Scope *> local_scopes_;
  Scope *global_scope_;
  std::unique_ptr<details::SSAGraphExecutor> executor_;

#ifdef PADDLE_WITH_CUDA
  std::unique_ptr<platform::NCCLContextMap> nccl_ctxs_;
#endif
};

ParallelExecutor::ParallelExecutor(
    size_t num_threads, bool use_event,
    const std::vector<platform::Place> &places,
    const std::unordered_set<std::string> &params,
    const ProgramDesc &startup_program, const ProgramDesc &main_program,
    const std::string &loss_var_name, Scope *scope)
    : member_(new ParallelExecutorPrivate(places)) {
  member_->global_scope_ = scope;

  // Step 1. RunStartupProgram and Bcast the params to devs.
  Executor exe(places[0]);
  exe.Run(startup_program, scope, 0);
  // Create local scopes
  for (size_t i = 0; i < member_->places_.size(); ++i) {
    member_->local_scopes_.push_back(&scope->NewScope());
  }

// Bcast Parameters to all GPUs
#ifdef PADDLE_WITH_CUDA
  member_->nccl_ctxs_.reset(new platform::NCCLContextMap(member_->places_));
#endif
  if (platform::is_gpu_place(places[0]) &&
      member_->local_scopes_.size() != 1) {  // Is CUDA
    BCastParamsToGPUs(startup_program);
  }
// Startup Program has been run. All local scopes has correct parameters.

// Step 2. Convert main_program to SSA form and dependency graph. Also, insert
// ncclOp
#ifdef PADDLE_WITH_CUDA
  details::MultiDevSSAGraphBuilder builder(member_->places_, loss_var_name,
                                           params, member_->local_scopes_,
                                           member_->nccl_ctxs_.get());
#else
  details::MultiDevSSAGraphBuilder builder(member_->places_, loss_var_name,
                                           params, member_->local_scopes_);
#endif
  auto graph = builder.Build(main_program);

  member_->executor_.reset(new details::ThreadedSSAGraphExecutor(
      num_threads, use_event, member_->local_scopes_, places,
      std::move(graph)));

  // Step 3. Create vars in each scope;
  for (auto *scope : member_->local_scopes_) {
    for (auto *var : main_program.Block(0).AllVars()) {
      if (scope->FindVar(var->Name()) != nullptr) {
        continue;
      }

      InitializeVariable(scope->Var(var->Name()), var->GetType());
    }
  }
}

void ParallelExecutor::BCastParamsToGPUs(
    const ProgramDesc &startup_program) const {
#ifdef PADDLE_WITH_CUDA
  auto *main_scope = member_->local_scopes_[0];

  for (auto *var_desc : startup_program.Block(0).AllVars()) {
    size_t idx = var_desc->Name().find("@GRAD");
    if (idx != std::string::npos) continue;
    if (var_desc->GetType() == proto::VarType::LOD_TENSOR) {
      auto &main_tensor =
          main_scope->FindVar(var_desc->Name())->Get<LoDTensor>();

      auto &dims = main_tensor.dims();

      if (paddle::platform::is_gpu_place(main_tensor.place())) {
        size_t numel = main_tensor.numel();
        ncclDataType_t data_type = platform::ToNCCLDataType(main_tensor.type());
        platform::NCCLGroupGuard guard;
        for (size_t i = 0; i < member_->places_.size(); ++i) {
          auto place = member_->places_[i];
          void *buffer;
          if (i == 0) {
            buffer = const_cast<void *>(main_tensor.data<void>());
          } else {
            auto local_scope = member_->local_scopes_[i];
            auto *t =
                local_scope->Var(var_desc->Name())->GetMutable<LoDTensor>();
            t->Resize(dims);
            buffer = t->mutable_data(place, main_tensor.type());
          }
          auto &nccl_ctx = member_->nccl_ctxs_->at(place);
          platform::dynload::ncclBcast(buffer, numel, data_type, 0,
                                       nccl_ctx.comm_, nccl_ctx.stream());
        }
      } else {
        platform::CPUPlace cpu;
        for (size_t i = 1; i < member_->places_.size(); ++i) {
          auto local_scope = member_->local_scopes_[i];
          auto *t = local_scope->Var(var_desc->Name())->GetMutable<LoDTensor>();
          t->Resize(dims);
          t->mutable_data(cpu, main_tensor.type());
          paddle::framework::TensorCopy(main_tensor, cpu, t);
        }
      }
    }
    member_->nccl_ctxs_->WaitAll();
  }
#else
  PADDLE_THROW("Not compiled with CUDA");
#endif
}

void ParallelExecutor::Run(const std::vector<std::string> &fetch_tensors,
                           const std::string &fetched_var_name) {
  auto fetch_data = member_->executor_->Run(fetch_tensors);
  *member_->global_scope_->Var(fetched_var_name)->GetMutable<FeedFetchList>() =
      fetch_data;
}

}  // namespace framework
}  // namespace paddle
<|endoftext|>


<commit_before>#include <map>

#include "EventAnalysis.h"

//================================================================================
// MATHEMATICAL AND ADDITIONAL FUNCTIONS
//================================================================================

// struct used for sorting vectors of hits (functor)

struct EntityComp {

  string property;
  EntityComp(string property) {this->property = property;}
  bool operator()(const Hit& h1, const Hit& h2) const {
    bool val = false;
    if(property == "TIME")
      val = h1.time < h2.time;
    else if (property == "EDEP")
      val = h1.edep < h2.edep;
    else if (property == "EVENTID")
      val = h1.eventID < h2.eventID;
    return val;
  }

};

void sort_hits(vector<Hit> &hits, string key) {

  sort(hits.begin(), hits.end(), EntityComp(key));

}

Hit add_hits(Hit &h1, Hit &h2, string winner="centroidWinner") {
  Hit h;
  h.eventID = h1.eventID;
  h.volumeID = h1.volumeID;
  h.trackID = h1.trackID;
  h.parentID = h1.parentID;
  h.primaryID = h1.primaryID;
  h.time = min(h1.time, h2.time);
  h.edep = h1.edep + h2.edep;
  if (winner == "centroidWinner") {
    h.posX = (h1.posX+h2.posX)/2.;
    h.posY = (h1.posY+h2.posY)/2.;
    h.posZ = (h1.posZ+h2.posZ)/2.;
  }
  else if (winner == "energyWinner") {
    if (h1.edep>=h2.edep) {
      h.posX = h1.posX;
      h.posY = h1.posY;
      h.posZ = h1.posZ;
    }
    else {
      h.posX = h2.posX;
      h.posY = h2.posY;
      h.posZ = h2.posZ;
    }
  }
  h.sourcePosX = h1.sourcePosX;
  h.sourcePosY = h1.sourcePosY;
  h.sourcePosZ = h1.sourcePosZ;
  h.nPhantomCompton = h1.nPhantomCompton;
  h.nCrystalCompton = h1.nCrystalCompton;
  return h;
}

//================================================================================
// DEFINITIONS OF CLASS FUNCTIONS - BASIC ANALYSIS
//================================================================================

EventAnalysis::EventAnalysis() {

  coincident_hits.clear();
  N0 = 0;
  N = 0;

}

void EventAnalysis::select_coincident_hits(vector<Hit> &hits) {

  N0 = hits.size();
  double COMPTON_E_TH = atof(getenv("GOJA_COMPTON_E_TH"))*1e3;
  for (unsigned int i=0; i<hits.size(); i++) {
    if (hits[i].edep>COMPTON_E_TH) coincident_hits.push_back(hits[i]);
  }
  N = coincident_hits.size();

}

void EventAnalysis::select_coincident_singles(vector<Hit> &hits) {

  if (DEBUG) {
    cout << "Hits:" << endl;
    for (unsigned int i=0; i<hits.size(); i++) {
      cout << '\t' << hits[i].volumeID << endl;
    }
    cout << '\t' << "# hits: " << hits.size() << endl;
  }

  map<int, Hit> singles;
  for (unsigned int i=0; i<hits.size(); i++) {
    int volumeID = hits[i].volumeID;
    if (singles.find(volumeID) == singles.end() ) {
      singles[volumeID] = hits[i];
    } else {
      singles[volumeID] = add_hits(singles[volumeID], hits[i]);
    }
  }
  N0 = singles.size();

  if (DEBUG) {
    cout << "Singles:" << endl;
    map<int, Hit>::iterator it = singles.begin();
    while(it != singles.end()) {
      cout << '\t' << it->first << endl;
      it++;
    }
    cout << '\t' << "# singles: " << singles.size() << endl;
  }

  double COMPTON_E_TH = atof(getenv("GOJA_COMPTON_E_TH"))*1e3;
  for (unsigned int i=0; i<singles.size(); i++) {
    if (singles[i].edep>COMPTON_E_TH) coincident_hits.push_back(singles[i]);
  }
  N = coincident_hits.size();
}

EventType EventAnalysis::verify_type_of_coincidence(Hit &h1, Hit &h2) {

  EventType t = kUnspecified;

  if (h1.eventID==h2.eventID) { //true, phantom-scattered and detector-scattered
    if (h1.nPhantomCompton==0 and h2.nPhantomCompton==0) {
      if (h1.nCrystalCompton==1 and h2.nCrystalCompton==1) { //true
        t = kTrue;
      }
      else { //detector-scattered
        t = kDetectorScattered;
      }
    }
    else { //phantom-scattered
      t = kPhantomScattered;
    }
  }
  else { //accidental
    t = kAccidental;
  }

  return t;

}

//================================================================================
// PRINTING
//================================================================================

void EventAnalysis::print_coincidences() {

  cout.setf(ios::fixed);

  Hit h1 = coincident_hits[0];
  Hit h2 = coincident_hits[1];

  cout.precision(2);
  cout << h1.posX/10. << "\t" << h1.posY/10. << "\t" << h1.posZ/10. << "\t";
  cout.precision(1);
  cout << h1.time << "\t";

  cout.precision(2);
  cout << h2.posX/10. << "\t" << h2.posY/10. << "\t" << h2.posZ/10. << "\t";
  cout.precision(1);
  cout << h2.time << "\t";

  cout << h1.volumeID << "\t";
  cout << h2.volumeID << "\t";

  cout.precision(2);
  cout << h1.edep << "\t";
  cout << h2.edep << "\t";

  cout << verify_type_of_coincidence(h1, h2) << "\t";

  cout.precision(2);
  cout << h1.sourcePosX/10. << "\t" << h1.sourcePosY/10. << "\t" << h1.sourcePosZ/10. << "\t";
  cout << h2.sourcePosX/10. << "\t" << h2.sourcePosY/10. << "\t" << h2.sourcePosZ/10. << endl;

}

//================================================================================
// MAIN ANALYSIS FUNCTION
//================================================================================

void EventAnalysis::analyze_event(vector<Hit> &hits, bool singles)
{

  int MAX_N = int(atof(getenv("GOJA_MAX_N")));
  int MAX_N0 = int(atof(getenv("GOJA_MAX_N0")));

  sort_hits(hits, "TIME");

  if (singles) {
    select_coincident_singles(hits);
  }
  else {
    select_coincident_hits(hits);
  }

  if (N==MAX_N and N0<=MAX_N0) print_coincidences();

  if (DEBUG) {
    cout.setf(ios::fixed);
    cout.precision(1);
    for (unsigned int i=0; i<hits.size(); i++) {
      double t = hits[i].time;
      if (i==0) cout << endl;
      cout.precision(1);
      cout << t << "\t";
      cout.precision(2);
      cout << hits[i].edep;
      if (i>0) {
        cout.precision(1);
        cout << "\t" << (hits[i].time-hits[i-1].time);
      }
      if (i==(hits.size()-1)) {
        cout.precision(1);
        cout << "\t" << (hits[i].time-hits[0].time) << endl;
        cout << N << "\t" << N0 << endl;
      }
      cout << endl;
    }
  }

}
<commit_msg>Adjust select_coincident_singles function to cylindricalPET systemType<commit_after>#include <map>
#include <string>
#include <sstream>

#include "EventAnalysis.h"

//================================================================================
// MATHEMATICAL AND ADDITIONAL FUNCTIONS
//================================================================================

// struct used for sorting vectors of hits (functor)

struct EntityComp {

  string property;
  EntityComp(string property) {this->property = property;}
  bool operator()(const Hit& h1, const Hit& h2) const {
    bool val = false;
    if(property == "TIME")
      val = h1.time < h2.time;
    else if (property == "EDEP")
      val = h1.edep < h2.edep;
    else if (property == "EVENTID")
      val = h1.eventID < h2.eventID;
    return val;
  }

};

void sort_hits(vector<Hit> &hits, string key) {

  sort(hits.begin(), hits.end(), EntityComp(key));

}

Hit add_hits(Hit &h1, Hit &h2, string winner="centroidWinner") {
  Hit h;
  h.eventID = h1.eventID;
  h.volumeID = h1.volumeID;
  h.trackID = h1.trackID;
  h.parentID = h1.parentID;
  h.primaryID = h1.primaryID;
  h.time = min(h1.time, h2.time);
  h.edep = h1.edep + h2.edep;
  if (winner == "centroidWinner") {
    h.posX = (h1.posX+h2.posX)/2.;
    h.posY = (h1.posY+h2.posY)/2.;
    h.posZ = (h1.posZ+h2.posZ)/2.;
  }
  else if (winner == "energyWinner") {
    if (h1.edep>=h2.edep) {
      h.posX = h1.posX;
      h.posY = h1.posY;
      h.posZ = h1.posZ;
    }
    else {
      h.posX = h2.posX;
      h.posY = h2.posY;
      h.posZ = h2.posZ;
    }
  }
  h.sourcePosX = h1.sourcePosX;
  h.sourcePosY = h1.sourcePosY;
  h.sourcePosZ = h1.sourcePosZ;
  h.nPhantomCompton = h1.nPhantomCompton;
  h.nCrystalCompton = h1.nCrystalCompton;
  return h;
}

//================================================================================
// DEFINITIONS OF CLASS FUNCTIONS - BASIC ANALYSIS
//================================================================================

EventAnalysis::EventAnalysis() {

  coincident_hits.clear();
  N0 = 0;
  N = 0;

}

void EventAnalysis::select_coincident_hits(vector<Hit> &hits) {

  N0 = hits.size();
  double COMPTON_E_TH = atof(getenv("GOJA_COMPTON_E_TH"))*1e3;
  for (unsigned int i=0; i<hits.size(); i++) {
    if (hits[i].edep>COMPTON_E_TH) coincident_hits.push_back(hits[i]);
  }
  N = coincident_hits.size();

}

void EventAnalysis::select_coincident_singles(vector<Hit> &hits) {

  map<string, Hit> singles;
  for (unsigned int i=0; i<hits.size(); i++) {
    stringstream IDss;
    string systemType = string(getenv("GOJA_SYSTEM_TYPE"));
    if (systemType == "scanner")
      IDss << hits[i].volumeID;
    else if (systemType == "cylindricalPET")
      IDss << hits[i].rsectorID << '_' << hits[i].layerID;
    string ID = IDss.str();
    if (singles.find(ID) == singles.end() ) {
      singles[ID] = hits[i];
    } else {
      singles[ID] = add_hits(singles[ID], hits[i]);
    }
  }
  N0 = singles.size();

  double COMPTON_E_TH = atof(getenv("GOJA_COMPTON_E_TH"))*1e3;
  map<string, Hit>::iterator it = singles.begin();
  while(it != singles.end()) {
    if ((it->second).edep>COMPTON_E_TH) coincident_hits.push_back(it->second);
    it++;
  }
  N = coincident_hits.size();

}

EventType EventAnalysis::verify_type_of_coincidence(Hit &h1, Hit &h2) {

  EventType t = kUnspecified;

  if (h1.eventID==h2.eventID) { //true, phantom-scattered and detector-scattered
    if (h1.nPhantomCompton==0 and h2.nPhantomCompton==0) {
      if (h1.nCrystalCompton==1 and h2.nCrystalCompton==1) { //true
        t = kTrue;
      }
      else { //detector-scattered
        t = kDetectorScattered;
      }
    }
    else { //phantom-scattered
      t = kPhantomScattered;
    }
  }
  else { //accidental
    t = kAccidental;
  }

  return t;

}

//================================================================================
// PRINTING
//================================================================================

void EventAnalysis::print_coincidences() {

  cout.setf(ios::fixed);

  Hit h1 = coincident_hits[0];
  Hit h2 = coincident_hits[1];

  cout.precision(2);
  cout << h1.posX/10. << "\t" << h1.posY/10. << "\t" << h1.posZ/10. << "\t";
  cout.precision(1);
  cout << h1.time << "\t";

  cout.precision(2);
  cout << h2.posX/10. << "\t" << h2.posY/10. << "\t" << h2.posZ/10. << "\t";
  cout.precision(1);
  cout << h2.time << "\t";

  cout << h1.volumeID << "\t";
  cout << h2.volumeID << "\t";

  cout.precision(2);
  cout << h1.edep << "\t";
  cout << h2.edep << "\t";

  cout << verify_type_of_coincidence(h1, h2) << "\t";

  cout.precision(2);
  cout << h1.sourcePosX/10. << "\t" << h1.sourcePosY/10. << "\t" << h1.sourcePosZ/10. << "\t";
  cout << h2.sourcePosX/10. << "\t" << h2.sourcePosY/10. << "\t" << h2.sourcePosZ/10. << endl;

}

//================================================================================
// MAIN ANALYSIS FUNCTION
//================================================================================

void EventAnalysis::analyze_event(vector<Hit> &hits, bool singles)
{

  int MAX_N = int(atof(getenv("GOJA_MAX_N")));
  int MAX_N0 = int(atof(getenv("GOJA_MAX_N0")));

  sort_hits(hits, "TIME");

  if (singles) {
    select_coincident_singles(hits);
  }
  else {
    select_coincident_hits(hits);
  }

  if (N==MAX_N and N0<=MAX_N0) print_coincidences();

  if (DEBUG) {
    cout.setf(ios::fixed);
    cout.precision(1);
    for (unsigned int i=0; i<hits.size(); i++) {
      double t = hits[i].time;
      if (i==0) cout << endl;
      cout.precision(1);
      cout << t << "\t";
      cout.precision(2);
      cout << hits[i].edep;
      if (i>0) {
        cout.precision(1);
        cout << "\t" << (hits[i].time-hits[i-1].time);
      }
      if (i==(hits.size()-1)) {
        cout.precision(1);
        cout << "\t" << (hits[i].time-hits[0].time) << endl;
        cout << N << "\t" << N0 << endl;
      }
      cout << endl;
    }
  }

}
<|endoftext|>