<commit_before>// -*- mode: c++; indent-tabs-mode: nil; tab-width: 2 -*-
// $Id: ExportInterface.cpp 3045 2010-04-05 13:07:29Z hieuhoang1972 $

/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2009 University of Edinburgh

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
***********************************************************************/

/**
 * Moses interface for main function, for single-threaded and multi-threaded.
 **/
#include <exception>
#include <fstream>
#include <sstream>
#include <vector>

#include "util/random.hh"
#include "util/usage.hh"

#ifdef WIN32
// Include Visual Leak Detector
//#include <vld.h>
#endif

#include "Hypothesis.h"
#include "Manager.h"
#include "StaticData.h"
#include "TypeDef.h"
#include "Util.h"
#include "Timer.h"
#include "TranslationModel/PhraseDictionary.h"
#include "FF/StatefulFeatureFunction.h"
#include "FF/StatelessFeatureFunction.h"
#include "TranslationTask.h"
#include "ExportInterface.h"

#ifdef HAVE_PROTOBUF
#include "hypergraph.pb.h"
#endif

#ifdef PT_UG
#include <boost/foreach.hpp>
#include "TranslationModel/UG/mmsapt.h"
#include "TranslationModel/UG/generic/program_options/ug_splice_arglist.h"
#endif

#include "ExportInterface.h"
#include "moses/server/Server.h"
using namespace std;
using namespace Moses;

namespace Moses
{
extern size_t g_numHypos;

void OutputFeatureWeightsForHypergraph(std::ostream &outputSearchGraphStream)
{
  outputSearchGraphStream.setf(std::ios::fixed);
  outputSearchGraphStream.precision(6);
  StaticData::Instance().GetAllWeights().Save(outputSearchGraphStream);
}
} //namespace Moses

SimpleTranslationInterface::SimpleTranslationInterface(const string &mosesIni): m_staticData(StaticData::Instance())
{
  if (!m_params.LoadParam(mosesIni)) {
    cerr << "Error; Cannot load parameters at " << mosesIni<<endl;
    exit(1);
  }
  if (!StaticData::LoadDataStatic(&m_params, mosesIni.c_str())) {
    cerr << "Error; Cannot load static data in file " << mosesIni<<endl;
    exit(1);
  }

  util::rand_init();

}

SimpleTranslationInterface::~SimpleTranslationInterface()
{}

//the simplified version of string input/output translation
string SimpleTranslationInterface::translate(const string &inputString)
{
  boost::shared_ptr<Moses::IOWrapper> ioWrapper(new IOWrapper);
  // main loop over set of input sentences
  size_t sentEnd = inputString.rfind('\n'); //find the last \n, the input stream has to be appended with \n to be translated
  const string &newString = sentEnd != string::npos ? inputString : inputString + '\n';

  istringstream inputStream(newString);  //create the stream for the interface
  ioWrapper->SetInputStreamFromString(inputStream);
  ostringstream outputStream;
  ioWrapper->SetOutputStream2SingleBestOutputCollector(&outputStream);

  boost::shared_ptr<InputType> source = ioWrapper->ReadInput();
  if (!source) return "Error: Source==null!!!";
  IFVERBOSE(1) {
    ResetUserTime();
  }

  // set up task of translating one sentence
  boost::shared_ptr<TranslationTask> task
  = TranslationTask::create(source, ioWrapper);
  task->Run();

  string output = outputStream.str();
  //now trim the end whitespace
  const string whitespace = " \t\f\v\n\r";
  size_t end = output.find_last_not_of(whitespace);
  return output.erase(end + 1);
}

Moses::StaticData& SimpleTranslationInterface::getStaticData()
{
  return StaticData::InstanceNonConst();
}
void SimpleTranslationInterface::DestroyFeatureFunctionStatic()
{
  FeatureFunction::Destroy();
}

Parameter params;

//! run moses in server mode
int
run_as_server()
{
#ifdef HAVE_XMLRPC_C
  MosesServer::Server server(params);
  return server.run(); // actually: don't return. see Server::run()
#else
  UTIL_THROW2("Moses was compiled without xmlrpc-c. "
              << "No server functionality available.");
#endif
}

int
batch_run()
{
  // shorthand for accessing information in StaticData
  const StaticData& staticData = StaticData::Instance();

  //initialise random numbers
  util::rand_init();

  IFVERBOSE(1) PrintUserTime("Created input-output object");

  // set up read/writing class:
  boost::shared_ptr<IOWrapper> ioWrapper(new IOWrapper);
  UTIL_THROW_IF2(ioWrapper == NULL, "Error; Failed to create IO object"
                 << " [" << HERE << "]");

  // check on weights
  const ScoreComponentCollection& weights = staticData.GetAllWeights();
  IFVERBOSE(2) {
    TRACE_ERR("The global weight vector looks like this: ");
    TRACE_ERR(weights);
    TRACE_ERR("\n");
  }

#ifdef WITH_THREADS
  ThreadPool pool(staticData.ThreadCount());
#endif

  // using context for adaptation:
  // e.g., context words / strings from config file / cmd line
  std::string context_string;
  params.SetParameter(context_string,"context-string",string(""));

  // ... or weights for documents/domains from config file / cmd. line
  std::string context_weights;
  params.SetParameter(context_weights,"context-weights",string(""));

  // ... or the surrounding context (--context-window ...)
  size_t size_t_max = std::numeric_limits<size_t>::max();
  bool use_context_window = ioWrapper->GetLookAhead() || ioWrapper->GetLookBack();
  bool use_context = use_context_window || context_string.size();
  bool use_sliding_context_window = (use_context_window
                                     && ioWrapper->GetLookAhead() != size_t_max);

  boost::shared_ptr<std::vector<std::string> >  context_window;
  boost::shared_ptr<std::vector<std::string> >* cw;
  cw = use_context_window ? &context_window : NULL;
  if (!cw && context_string.size())
    context_window.reset(new std::vector<std::string>(1,context_string));

  // global scope of caches, biases, etc., if any
  boost::shared_ptr<ContextScope> gscope;
  if (!use_sliding_context_window)
    gscope.reset(new ContextScope);

  // main loop over set of input sentences
  boost::shared_ptr<InputType> source;
  while ((source = ioWrapper->ReadInput(cw)) != NULL) {
    IFVERBOSE(1) ResetUserTime();

    // set up task of translating one sentence
    boost::shared_ptr<ContextScope>  lscope;
    if (gscope) lscope = gscope;
    else lscope.reset(new ContextScope);

    boost::shared_ptr<TranslationTask> task;
    task = TranslationTask::create(source, ioWrapper, lscope);

    if (cw) {
      if (context_string.size())
        context_window->push_back(context_string);
      if(!use_sliding_context_window)
        cw = NULL;
    }

    if (context_window)
      task->SetContextWindow(context_window);

    if (context_weights != "" && !task->GetScope()->GetContextWeights())
      task->GetScope()->SetContextWeights(context_weights);

    // Allow for (sentence-)context-specific processing prior to
    // decoding. This can be used, for example, for context-sensitive
    // phrase lookup.
    FeatureFunction::SetupAll(*task);

    // execute task
#ifdef WITH_THREADS
#ifdef PT_UG
    // simulated post-editing requires threads (within the dynamic phrase tables)
    // but runs all sentences serially, to allow updating of the bitext.
    bool spe = params.isParamSpecified("spe-src");
    if (spe) {
      // simulated post-editing: always run single-threaded!
      task->Run();
      string src,trg,aln;
      UTIL_THROW_IF2(!getline(*ioWrapper->spe_src,src), "[" << HERE << "] "
                     << "missing update data for simulated post-editing.");
      UTIL_THROW_IF2(!getline(*ioWrapper->spe_trg,trg), "[" << HERE << "] "
                     << "missing update data for simulated post-editing.");
      UTIL_THROW_IF2(!getline(*ioWrapper->spe_aln,aln), "[" << HERE << "] "
                     << "missing update data for simulated post-editing.");
      BOOST_FOREACH (PhraseDictionary* pd, PhraseDictionary::GetColl()) {
        Mmsapt* sapt = dynamic_cast<Mmsapt*>(pd);
        if (sapt) sapt->add(src,trg,aln);
        VERBOSE(1,"[" << HERE << " added src] " << src << endl);
        VERBOSE(1,"[" << HERE << " added trg] " << trg << endl);
        VERBOSE(1,"[" << HERE << " added aln] " << aln << endl);
      }
    } else pool.Submit(task);
#else
    pool.Submit(task);

#endif
#else
    task->Run();
#endif
  }

  // we are done, finishing up
#ifdef WITH_THREADS
  pool.Stop(true); //flush remaining jobs
#endif

  FeatureFunction::Destroy();

  IFVERBOSE(1) util::PrintUsage(std::cerr);

  cerr << "g_numHypos=" << Moses::g_numHypos << endl;

#ifndef EXIT_RETURN
  //This avoids that destructors are called (it can take a long time)
  exit(EXIT_SUCCESS);
#else
  return EXIT_SUCCESS;
#endif
}

/** Called by main function of the command line version of the decoder **/
int decoder_main(int argc, char** argv)
{
#ifdef NDEBUG
  try
#endif
  {
#ifdef HAVE_PROTOBUF
    GOOGLE_PROTOBUF_VERIFY_VERSION;
#endif

    // echo command line, if verbose
    IFVERBOSE(1) {
      TRACE_ERR("command: ");
      for(int i=0; i<argc; ++i) TRACE_ERR(argv[i]<<" ");
      TRACE_ERR(endl);
    }

    // set number of significant decimals in output
    FixPrecision(cout);
    FixPrecision(cerr);

    // load all the settings into the Parameter class
    // (stores them as strings, or array of strings)
    if (!params.LoadParam(argc,argv))
      exit(1);

    // initialize all "global" variables, which are stored in StaticData
    // note: this also loads models such as the language model, etc.
    if (!StaticData::LoadDataStatic(&params, argv[0]))
      exit(1);

    // setting "-show-weights" -> just dump out weights and exit
    if (params.isParamSpecified("show-weights")) {
      ShowWeights();
      exit(0);
    }

    if (params.GetParam("server"))
      return run_as_server();
    else
      return batch_run();

  }
#ifdef NDEBUG
  catch (const std::exception &e) {
    std::cerr << "Exception: " << e.what() << std::endl;
    return EXIT_FAILURE;
  }
#endif
}

<commit_msg>print out num of hypos created before segfaulting<commit_after>// -*- mode: c++; indent-tabs-mode: nil; tab-width: 2 -*-
// $Id: ExportInterface.cpp 3045 2010-04-05 13:07:29Z hieuhoang1972 $

/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2009 University of Edinburgh

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
***********************************************************************/

/**
 * Moses interface for main function, for single-threaded and multi-threaded.
 **/
#include <exception>
#include <fstream>
#include <sstream>
#include <vector>

#include "util/random.hh"
#include "util/usage.hh"

#ifdef WIN32
// Include Visual Leak Detector
//#include <vld.h>
#endif

#include "Hypothesis.h"
#include "Manager.h"
#include "StaticData.h"
#include "TypeDef.h"
#include "Util.h"
#include "Timer.h"
#include "TranslationModel/PhraseDictionary.h"
#include "FF/StatefulFeatureFunction.h"
#include "FF/StatelessFeatureFunction.h"
#include "TranslationTask.h"
#include "ExportInterface.h"

#ifdef HAVE_PROTOBUF
#include "hypergraph.pb.h"
#endif

#ifdef PT_UG
#include <boost/foreach.hpp>
#include "TranslationModel/UG/mmsapt.h"
#include "TranslationModel/UG/generic/program_options/ug_splice_arglist.h"
#endif

#include "ExportInterface.h"
#include "moses/server/Server.h"
using namespace std;
using namespace Moses;

namespace Moses
{
extern size_t g_numHypos;

void OutputFeatureWeightsForHypergraph(std::ostream &outputSearchGraphStream)
{
  outputSearchGraphStream.setf(std::ios::fixed);
  outputSearchGraphStream.precision(6);
  StaticData::Instance().GetAllWeights().Save(outputSearchGraphStream);
}
} //namespace Moses

SimpleTranslationInterface::SimpleTranslationInterface(const string &mosesIni): m_staticData(StaticData::Instance())
{
  if (!m_params.LoadParam(mosesIni)) {
    cerr << "Error; Cannot load parameters at " << mosesIni<<endl;
    exit(1);
  }
  if (!StaticData::LoadDataStatic(&m_params, mosesIni.c_str())) {
    cerr << "Error; Cannot load static data in file " << mosesIni<<endl;
    exit(1);
  }

  util::rand_init();

}

SimpleTranslationInterface::~SimpleTranslationInterface()
{}

//the simplified version of string input/output translation
string SimpleTranslationInterface::translate(const string &inputString)
{
  boost::shared_ptr<Moses::IOWrapper> ioWrapper(new IOWrapper);
  // main loop over set of input sentences
  size_t sentEnd = inputString.rfind('\n'); //find the last \n, the input stream has to be appended with \n to be translated
  const string &newString = sentEnd != string::npos ? inputString : inputString + '\n';

  istringstream inputStream(newString);  //create the stream for the interface
  ioWrapper->SetInputStreamFromString(inputStream);
  ostringstream outputStream;
  ioWrapper->SetOutputStream2SingleBestOutputCollector(&outputStream);

  boost::shared_ptr<InputType> source = ioWrapper->ReadInput();
  if (!source) return "Error: Source==null!!!";
  IFVERBOSE(1) {
    ResetUserTime();
  }

  // set up task of translating one sentence
  boost::shared_ptr<TranslationTask> task
  = TranslationTask::create(source, ioWrapper);
  task->Run();

  string output = outputStream.str();
  //now trim the end whitespace
  const string whitespace = " \t\f\v\n\r";
  size_t end = output.find_last_not_of(whitespace);
  return output.erase(end + 1);
}

Moses::StaticData& SimpleTranslationInterface::getStaticData()
{
  return StaticData::InstanceNonConst();
}
void SimpleTranslationInterface::DestroyFeatureFunctionStatic()
{
  FeatureFunction::Destroy();
}

Parameter params;

//! run moses in server mode
int
run_as_server()
{
#ifdef HAVE_XMLRPC_C
  MosesServer::Server server(params);
  return server.run(); // actually: don't return. see Server::run()
#else
  UTIL_THROW2("Moses was compiled without xmlrpc-c. "
              << "No server functionality available.");
#endif
}

int
batch_run()
{
  // shorthand for accessing information in StaticData
  const StaticData& staticData = StaticData::Instance();

  //initialise random numbers
  util::rand_init();

  IFVERBOSE(1) PrintUserTime("Created input-output object");

  // set up read/writing class:
  boost::shared_ptr<IOWrapper> ioWrapper(new IOWrapper);
  UTIL_THROW_IF2(ioWrapper == NULL, "Error; Failed to create IO object"
                 << " [" << HERE << "]");

  // check on weights
  const ScoreComponentCollection& weights = staticData.GetAllWeights();
  IFVERBOSE(2) {
    TRACE_ERR("The global weight vector looks like this: ");
    TRACE_ERR(weights);
    TRACE_ERR("\n");
  }

#ifdef WITH_THREADS
  ThreadPool pool(staticData.ThreadCount());
#endif

  // using context for adaptation:
  // e.g., context words / strings from config file / cmd line
  std::string context_string;
  params.SetParameter(context_string,"context-string",string(""));

  // ... or weights for documents/domains from config file / cmd. line
  std::string context_weights;
  params.SetParameter(context_weights,"context-weights",string(""));

  // ... or the surrounding context (--context-window ...)
  size_t size_t_max = std::numeric_limits<size_t>::max();
  bool use_context_window = ioWrapper->GetLookAhead() || ioWrapper->GetLookBack();
  bool use_context = use_context_window || context_string.size();
  bool use_sliding_context_window = (use_context_window
                                     && ioWrapper->GetLookAhead() != size_t_max);

  boost::shared_ptr<std::vector<std::string> >  context_window;
  boost::shared_ptr<std::vector<std::string> >* cw;
  cw = use_context_window ? &context_window : NULL;
  if (!cw && context_string.size())
    context_window.reset(new std::vector<std::string>(1,context_string));

  // global scope of caches, biases, etc., if any
  boost::shared_ptr<ContextScope> gscope;
  if (!use_sliding_context_window)
    gscope.reset(new ContextScope);

  // main loop over set of input sentences
  boost::shared_ptr<InputType> source;
  while ((source = ioWrapper->ReadInput(cw)) != NULL) {
    IFVERBOSE(1) ResetUserTime();

    // set up task of translating one sentence
    boost::shared_ptr<ContextScope>  lscope;
    if (gscope) lscope = gscope;
    else lscope.reset(new ContextScope);

    boost::shared_ptr<TranslationTask> task;
    task = TranslationTask::create(source, ioWrapper, lscope);

    if (cw) {
      if (context_string.size())
        context_window->push_back(context_string);
      if(!use_sliding_context_window)
        cw = NULL;
    }

    if (context_window)
      task->SetContextWindow(context_window);

    if (context_weights != "" && !task->GetScope()->GetContextWeights())
      task->GetScope()->SetContextWeights(context_weights);

    // Allow for (sentence-)context-specific processing prior to
    // decoding. This can be used, for example, for context-sensitive
    // phrase lookup.
    FeatureFunction::SetupAll(*task);

    // execute task
#ifdef WITH_THREADS
#ifdef PT_UG
    // simulated post-editing requires threads (within the dynamic phrase tables)
    // but runs all sentences serially, to allow updating of the bitext.
    bool spe = params.isParamSpecified("spe-src");
    if (spe) {
      // simulated post-editing: always run single-threaded!
      task->Run();
      string src,trg,aln;
      UTIL_THROW_IF2(!getline(*ioWrapper->spe_src,src), "[" << HERE << "] "
                     << "missing update data for simulated post-editing.");
      UTIL_THROW_IF2(!getline(*ioWrapper->spe_trg,trg), "[" << HERE << "] "
                     << "missing update data for simulated post-editing.");
      UTIL_THROW_IF2(!getline(*ioWrapper->spe_aln,aln), "[" << HERE << "] "
                     << "missing update data for simulated post-editing.");
      BOOST_FOREACH (PhraseDictionary* pd, PhraseDictionary::GetColl()) {
        Mmsapt* sapt = dynamic_cast<Mmsapt*>(pd);
        if (sapt) sapt->add(src,trg,aln);
        VERBOSE(1,"[" << HERE << " added src] " << src << endl);
        VERBOSE(1,"[" << HERE << " added trg] " << trg << endl);
        VERBOSE(1,"[" << HERE << " added aln] " << aln << endl);
      }
    } else pool.Submit(task);
#else
    pool.Submit(task);

#endif
#else
    task->Run();
#endif
  }

  // we are done, finishing up
#ifdef WITH_THREADS
  pool.Stop(true); //flush remaining jobs
#endif

  cerr << "g_numHypos=" << Moses::g_numHypos << endl;

  FeatureFunction::Destroy();

  IFVERBOSE(1) util::PrintUsage(std::cerr);

#ifndef EXIT_RETURN
  //This avoids that destructors are called (it can take a long time)
  exit(EXIT_SUCCESS);
#else
  return EXIT_SUCCESS;
#endif
}

/** Called by main function of the command line version of the decoder **/
int decoder_main(int argc, char** argv)
{
#ifdef NDEBUG
  try
#endif
  {
#ifdef HAVE_PROTOBUF
    GOOGLE_PROTOBUF_VERIFY_VERSION;
#endif

    // echo command line, if verbose
    IFVERBOSE(1) {
      TRACE_ERR("command: ");
      for(int i=0; i<argc; ++i) TRACE_ERR(argv[i]<<" ");
      TRACE_ERR(endl);
    }

    // set number of significant decimals in output
    FixPrecision(cout);
    FixPrecision(cerr);

    // load all the settings into the Parameter class
    // (stores them as strings, or array of strings)
    if (!params.LoadParam(argc,argv))
      exit(1);

    // initialize all "global" variables, which are stored in StaticData
    // note: this also loads models such as the language model, etc.
    if (!StaticData::LoadDataStatic(&params, argv[0]))
      exit(1);

    // setting "-show-weights" -> just dump out weights and exit
    if (params.isParamSpecified("show-weights")) {
      ShowWeights();
      exit(0);
    }

    if (params.GetParam("server"))
      return run_as_server();
    else
      return batch_run();

  }
#ifdef NDEBUG
  catch (const std::exception &e) {
    std::cerr << "Exception: " << e.what() << std::endl;
    return EXIT_FAILURE;
  }
#endif
}

<|endoftext|>


<commit_before>#ifndef MESSAGE_QUEUE_HPP
#define MESSAGE_QUEUE_HPP

#include "cppa/message.hpp"
#include "cppa/ref_counted.hpp"

namespace cppa {

// forward declaration
class invoke_rules;

class message_queue : public ref_counted
{

 protected:

    bool m_trap_exit;
    message m_last_dequeued;

 public:

    message_queue();

    virtual void enqueue(const message&) = 0;
    virtual const message& dequeue() = 0;
    virtual void dequeue(invoke_rules&) = 0;
    virtual bool try_dequeue(message&) = 0;
    virtual bool try_dequeue(invoke_rules&) = 0;

    inline bool trap_exit() const;
    inline void trap_exit(bool value);
    inline const message& last_dequeued();

};

inline bool message_queue::trap_exit() const
{
    return m_trap_exit;
}

inline void message_queue::trap_exit(bool value)
{
    m_trap_exit = value;
}

inline const message& message_queue::last_dequeued()
{
    return m_last_dequeued;
}

} // namespace cppa

#endif // MESSAGE_QUEUE_HPP
<commit_msg>const correctness<commit_after>#ifndef MESSAGE_QUEUE_HPP
#define MESSAGE_QUEUE_HPP

#include "cppa/message.hpp"
#include "cppa/ref_counted.hpp"

namespace cppa {

// forward declaration
class invoke_rules;

class message_queue : public ref_counted
{

 protected:

    bool m_trap_exit;
    message m_last_dequeued;

 public:

    message_queue();

    virtual void enqueue(const message&) = 0;
    virtual const message& dequeue() = 0;
    virtual void dequeue(invoke_rules&) = 0;
    virtual bool try_dequeue(message&) = 0;
    virtual bool try_dequeue(invoke_rules&) = 0;

    inline bool trap_exit() const;
    inline void trap_exit(bool value);
    inline const message& last_dequeued() const;

};

inline bool message_queue::trap_exit() const
{
    return m_trap_exit;
}

inline void message_queue::trap_exit(bool value)
{
    m_trap_exit = value;
}

inline const message& message_queue::last_dequeued() const
{
    return m_last_dequeued;
}

} // namespace cppa

#endif // MESSAGE_QUEUE_HPP
<|endoftext|>


<commit_before>/*************************************************************************
 *
 *  $RCSfile: test_sec.cxx,v $
 *
 *  $Revision: 1.2 $
 *
 *  last change: $Author: jl $ $Date: 2001-03-19 10:10:07 $
 *
 *  The Contents of this file are made available subject to the terms of
 *  either of the following licenses
 *
 *         - GNU Lesser General Public License Version 2.1
 *         - Sun Industry Standards Source License Version 1.1
 *
 *  Sun Microsystems Inc., October, 2000
 *
 *  GNU Lesser General Public License Version 2.1
 *  =============================================
 *  Copyright 2000 by Sun Microsystems, Inc.
 *  901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License version 2.1, as published by the Free Software Foundation.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *  MA  02111-1307  USA
 *
 *
 *  Sun Industry Standards Source License Version 1.1
 *  =================================================
 *  The contents of this file are subject to the Sun Industry Standards
 *  Source License Version 1.1 (the "License"); You may not use this file
 *  except in compliance with the License. You may obtain a copy of the
 *  License at http://www.openoffice.org/license.html.
 *
 *  Software provided under this License is provided on an "AS IS" basis,
 *  WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
 *  WITHOUT LIMITATION, WARRANTIES THAT THE SOFTWARE IS FREE OF DEFECTS,
 *  MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE, OR NON-INFRINGING.
 *  See the License for the specific provisions governing your rights and
 *  obligations concerning the Software.
 *
 *  The Initial Developer of the Original Code is: Sun Microsystems, Inc.
 *
 *  Copyright: 2000 by Sun Microsystems, Inc.
 *
 *  All Rights Reserved.
 *
 *  Contributor(s): _______________________________________
 *
 *
 ************************************************************************/

#include <stdio.h>

#include <uno/current_context.hxx>

#include <cppuhelper/implbase1.hxx>

#include <com/sun/star/lang/XMultiServiceFactory.hpp>
#include <com/sun/star/lang/XSingleServiceFactory.hpp>
#include <com/sun/star/loader/XImplementationLoader.hpp>
#include <com/sun/star/container/XNameContainer.hpp>

using namespace rtl;
using namespace cppu;
using namespace com::sun::star::uno;
using namespace com::sun::star::security;
using namespace com::sun::star::lang;
using namespace com::sun::star::loader;
using namespace com::sun::star::registry;
using namespace com::sun::star::container;

static inline void out( const sal_Char * p )
{
    ::fprintf( stderr, p );
}
static inline void out( const OUString & r )
{
    OString aStr( OUStringToOString( r, RTL_TEXTENCODING_ASCII_US ) );
    out( aStr.getStr() );
}

static Reference< XInterface > load(
    const Reference< XMultiServiceFactory > & xMgr,
    const char * service,
    const char * implName, const char * activator, const char * loc ) throw ()
{
    Reference< XInterface > xRet(
        xMgr->createInstance( OUString::createFromAscii( service ) ) );
    if (xRet.is())
        return xRet;

    try
    {
        Reference< XImplementationLoader > xLoader(
            xMgr->createInstance( OUString::createFromAscii( activator ) ), UNO_QUERY );
        if (xLoader.is())
        {
            Reference< XSingleServiceFactory > xFac( xLoader->activate(
                OUString::createFromAscii( implName ), OUString(),
                OUString::createFromAscii( loc ),
                Reference< XRegistryKey >() ), UNO_QUERY );
            if (xFac.is())
                return xFac->createInstance();
        }
        else
        {
            out( "### cannot activate loader \"" );
            out( activator );
            out( "\"!\n" );
        }
    }
    catch (Exception &)
    {
        out( "### cannot activate service \"" );
        out( service );
        out( "\"!\n" );
    }
    return Reference< XInterface >();
}

static void setEnv( const Reference< XMultiServiceFactory > & xMgr ) throw ()
{
    OSL_ASSERT( xMgr.is() );
    Reference< XNameContainer > xContext( getCurrentContext(), UNO_QUERY );
    OSL_ASSERT( xContext.is() );

    try
    {
        xContext->insertByName(
            OUString( RTL_CONSTASCII_USTRINGPARAM("com.sun.star.lang.ServiceManager") ),
            makeAny( xMgr ) );
        xContext->insertByName(
            OUString( RTL_CONSTASCII_USTRINGPARAM("com.sun.star.security.AccessController") ),
            makeAny( load( xMgr,
                           "com.sun.star.security.AccessController",
                           "com.sun.star.comp.security.AccessController",
                           "com.sun.star.loader.Java2",
                           "com.sun.star.comp.security.AccessController" ) ) );
        xContext->insertByName(
            OUString( RTL_CONSTASCII_USTRINGPARAM("com.sun.star.security.auth.login.LoginContext") ),
            makeAny( load( xMgr,
                           "com.sun.star.security.auth.login.LoginContext",
                           "com.sun.star.comp.security.auth.login.LoginContext",
                           "com.sun.star.loader.Java2",
                           "com.sun.star.comp.security.auth.login.LoginContext" ) ) );
    }
    catch (Exception & rExc)
    {
        out( "### exception occured: " );
        out( rExc.Message );
        out( "\n" );
    }
}

//##################################################################################################
//##################################################################################################
//##################################################################################################

struct PrivAction : WeakImplHelper1< XPrivilegedAction >
{
    virtual Any SAL_CALL run()
        throw (Exception)
    {
        throw IllegalArgumentException(
            OUString::createFromAscii("testtest"), (OWeakObject *)this, (sal_Int16)5 );
    }
};

struct Thread1 : public OThread
{
    Reference< XMultiServiceFactory > _xMgr;

    Thread1( const Reference< XMultiServiceFactory > & xMgr )
        : _xMgr( xMgr )
        {}

    void f()
    {
        Reference< XNameAccess > xAccess( getCurrentContext(), UNO_QUERY );
        OSL_ASSERT( xAccess->getByName( OUString::createFromAscii("a") ) == (sal_Int16)5 );
        Reference< XNameContainer > xCont( getCurrentContext(), UNO_QUERY );
        xCont->insertByName( OUString::createFromAscii("b"), makeAny( (sal_Int32)6 ) );
    }
    virtual void SAL_CALL run()
    {
        try
        {
            setEnv( _xMgr );
            Reference< XNameContainer > xCont( getCurrentContext(), UNO_QUERY );
            xCont->insertByName( OUString::createFromAscii("a"), makeAny( (sal_Int32)5 ) );
            f();
            Reference< XNameAccess > xAccess( getCurrentContext(), UNO_QUERY );
            OSL_ASSERT( xAccess->getByName( OUString::createFromAscii("b") ) == (sal_Int16)6 );

//              checkPermission(
//                  Permission( OUString::createFromAscii("java.io.FilePermission"),
//                              OUString::createFromAscii("f:\\userprofiles.dat"),
//                              OUString::createFromAscii("read") ) );

//              try
//              {
//                  Reference< XCurrentContext > xContext( getCurrentContext() );
//                  Reference< XAccessController > xACC( xContext->getAccessController() );
//                  xACC->doPrivileged( new PrivAction(), Reference< XAccessControlContext >() );
//              }
//              catch (IllegalArgumentException & r)
//              {
//                  OSL_ASSERT( r.ArgumentPosition == 5 &&
//                              r.Message.equalsAsciiL( RTL_CONSTASCII_STRINGPARAM("testtest") ));
//              }
        }
        catch (RuntimeException & rExc)
        {
            out( rExc.Message );
        }
    }
};

void test_security( const Reference< XMultiServiceFactory > & xMgr ) throw ()
{
    setEnv( xMgr );
    OSL_ASSERT( getCurrentContext()->getServiceManager().is() );

    Thread1 thread1( xMgr );
    thread1.create();
    thread1.join();
    out( "> exiting...\n" );
}
<commit_msg>INTEGRATION: CWS ooo19126 (1.2.190); FILE MERGED 2005/09/05 13:54:38 rt 1.2.190.1: #i54170# Change license header: remove SISSL<commit_after>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: test_sec.cxx,v $
 *
 *  $Revision: 1.3 $
 *
 *  last change: $Author: rt $ $Date: 2005-09-08 08:59:16 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#include <stdio.h>

#include <uno/current_context.hxx>

#include <cppuhelper/implbase1.hxx>

#include <com/sun/star/lang/XMultiServiceFactory.hpp>
#include <com/sun/star/lang/XSingleServiceFactory.hpp>
#include <com/sun/star/loader/XImplementationLoader.hpp>
#include <com/sun/star/container/XNameContainer.hpp>

using namespace rtl;
using namespace cppu;
using namespace com::sun::star::uno;
using namespace com::sun::star::security;
using namespace com::sun::star::lang;
using namespace com::sun::star::loader;
using namespace com::sun::star::registry;
using namespace com::sun::star::container;

static inline void out( const sal_Char * p )
{
    ::fprintf( stderr, p );
}
static inline void out( const OUString & r )
{
    OString aStr( OUStringToOString( r, RTL_TEXTENCODING_ASCII_US ) );
    out( aStr.getStr() );
}

static Reference< XInterface > load(
    const Reference< XMultiServiceFactory > & xMgr,
    const char * service,
    const char * implName, const char * activator, const char * loc ) throw ()
{
    Reference< XInterface > xRet(
        xMgr->createInstance( OUString::createFromAscii( service ) ) );
    if (xRet.is())
        return xRet;

    try
    {
        Reference< XImplementationLoader > xLoader(
            xMgr->createInstance( OUString::createFromAscii( activator ) ), UNO_QUERY );
        if (xLoader.is())
        {
            Reference< XSingleServiceFactory > xFac( xLoader->activate(
                OUString::createFromAscii( implName ), OUString(),
                OUString::createFromAscii( loc ),
                Reference< XRegistryKey >() ), UNO_QUERY );
            if (xFac.is())
                return xFac->createInstance();
        }
        else
        {
            out( "### cannot activate loader \"" );
            out( activator );
            out( "\"!\n" );
        }
    }
    catch (Exception &)
    {
        out( "### cannot activate service \"" );
        out( service );
        out( "\"!\n" );
    }
    return Reference< XInterface >();
}

static void setEnv( const Reference< XMultiServiceFactory > & xMgr ) throw ()
{
    OSL_ASSERT( xMgr.is() );
    Reference< XNameContainer > xContext( getCurrentContext(), UNO_QUERY );
    OSL_ASSERT( xContext.is() );

    try
    {
        xContext->insertByName(
            OUString( RTL_CONSTASCII_USTRINGPARAM("com.sun.star.lang.ServiceManager") ),
            makeAny( xMgr ) );
        xContext->insertByName(
            OUString( RTL_CONSTASCII_USTRINGPARAM("com.sun.star.security.AccessController") ),
            makeAny( load( xMgr,
                           "com.sun.star.security.AccessController",
                           "com.sun.star.comp.security.AccessController",
                           "com.sun.star.loader.Java2",
                           "com.sun.star.comp.security.AccessController" ) ) );
        xContext->insertByName(
            OUString( RTL_CONSTASCII_USTRINGPARAM("com.sun.star.security.auth.login.LoginContext") ),
            makeAny( load( xMgr,
                           "com.sun.star.security.auth.login.LoginContext",
                           "com.sun.star.comp.security.auth.login.LoginContext",
                           "com.sun.star.loader.Java2",
                           "com.sun.star.comp.security.auth.login.LoginContext" ) ) );
    }
    catch (Exception & rExc)
    {
        out( "### exception occured: " );
        out( rExc.Message );
        out( "\n" );
    }
}

//##################################################################################################
//##################################################################################################
//##################################################################################################

struct PrivAction : WeakImplHelper1< XPrivilegedAction >
{
    virtual Any SAL_CALL run()
        throw (Exception)
    {
        throw IllegalArgumentException(
            OUString::createFromAscii("testtest"), (OWeakObject *)this, (sal_Int16)5 );
    }
};

struct Thread1 : public OThread
{
    Reference< XMultiServiceFactory > _xMgr;

    Thread1( const Reference< XMultiServiceFactory > & xMgr )
        : _xMgr( xMgr )
        {}

    void f()
    {
        Reference< XNameAccess > xAccess( getCurrentContext(), UNO_QUERY );
        OSL_ASSERT( xAccess->getByName( OUString::createFromAscii("a") ) == (sal_Int16)5 );
        Reference< XNameContainer > xCont( getCurrentContext(), UNO_QUERY );
        xCont->insertByName( OUString::createFromAscii("b"), makeAny( (sal_Int32)6 ) );
    }
    virtual void SAL_CALL run()
    {
        try
        {
            setEnv( _xMgr );
            Reference< XNameContainer > xCont( getCurrentContext(), UNO_QUERY );
            xCont->insertByName( OUString::createFromAscii("a"), makeAny( (sal_Int32)5 ) );
            f();
            Reference< XNameAccess > xAccess( getCurrentContext(), UNO_QUERY );
            OSL_ASSERT( xAccess->getByName( OUString::createFromAscii("b") ) == (sal_Int16)6 );

//              checkPermission(
//                  Permission( OUString::createFromAscii("java.io.FilePermission"),
//                              OUString::createFromAscii("f:\\userprofiles.dat"),
//                              OUString::createFromAscii("read") ) );

//              try
//              {
//                  Reference< XCurrentContext > xContext( getCurrentContext() );
//                  Reference< XAccessController > xACC( xContext->getAccessController() );
//                  xACC->doPrivileged( new PrivAction(), Reference< XAccessControlContext >() );
//              }
//              catch (IllegalArgumentException & r)
//              {
//                  OSL_ASSERT( r.ArgumentPosition == 5 &&
//                              r.Message.equalsAsciiL( RTL_CONSTASCII_STRINGPARAM("testtest") ));
//              }
        }
        catch (RuntimeException & rExc)
        {
            out( rExc.Message );
        }
    }
};

void test_security( const Reference< XMultiServiceFactory > & xMgr ) throw ()
{
    setEnv( xMgr );
    OSL_ASSERT( getCurrentContext()->getServiceManager().is() );

    Thread1 thread1( xMgr );
    thread1.create();
    thread1.join();
    out( "> exiting...\n" );
}
<|endoftext|>


<commit_before>#include "datastream_ROS.h"
#include <QTextStream>
#include <QFile>
#include <QMessageBox>
#include <QDebug>
#include <thread>
#include <mutex>
#include <chrono>
#include <thread>
#include <QProgressDialog>
#include <QtGlobal>
#include <QApplication>
#include <QProcess>
#include <QCheckBox>
#include <QSettings>
#include <QFileDialog>
#include <ros/callback_queue.h>
#include <rosbag/bag.h>
#include <topic_tools/shape_shifter.h>
#include <ros/transport_hints.h>

#include "../dialog_select_ros_topics.h"
#include "../rule_editing.h"
#include "../qnodedialog.h"
#include "../shape_shifter_factory.hpp"

DataStreamROS::DataStreamROS():
    DataStreamer(),
    _node(nullptr),
    _destination_data(nullptr),
    _action_saveIntoRosbag(nullptr),
    _prev_clock_time(0)
{
    _running = false;
    _periodic_timer = new QTimer();
    connect( _periodic_timer, &QTimer::timeout,
             this, &DataStreamROS::timerCallback);

    loadDefaultSettings();
}

void DataStreamROS::topicCallback(const topic_tools::ShapeShifter::ConstPtr& msg,
                                  const std::string &topic_name)
{
    if( !_running ){
        return;
    }

    using namespace RosIntrospection;
    const auto&  md5sum     =  msg->getMD5Sum();
    const auto&  datatype   =  msg->getDataType();
    const auto&  definition =  msg->getMessageDefinition() ;

    // register the message type
    _ros_parser.registerSchema( topic_name, md5sum,
                                RosIntrospection::ROSType(datatype),
                                definition);

    RosIntrospectionFactory::registerMessage(topic_name, md5sum, datatype, definition );

    //------------------------------------

    // it is more efficient to recycle this elements
    static std::vector<uint8_t> buffer;

    buffer.resize( msg->size() );

    ros::serialization::OStream stream(buffer.data(), buffer.size());
    msg->write(stream);

    double msg_time = ros::Time::now().toSec();
    if( msg_time == 0)
    {
      // corner case: use_sim_time == true but topic /clock is not published
      msg_time = ros::WallTime::now().toSec();
    }

    if( msg_time < _prev_clock_time )
    {
        // clear
        for (auto& it: dataMap().numeric ) {
            it.second.clear();
        }
        for (auto& it: dataMap().user_defined ){
            it.second.clear();
        }
        emit clearBuffers();
    }
    _prev_clock_time = msg_time;

    MessageRef buffer_view( buffer );
    _ros_parser.pushMessageRef( topic_name, buffer_view, msg_time );

    std::lock_guard<std::mutex> lock( mutex() );
    const std::string prefixed_topic_name = _prefix + topic_name;

    // adding raw serialized msg for future uses.
    // do this before msg_time normalization
    {
        auto plot_pair = dataMap().user_defined.find( prefixed_topic_name );
        if( plot_pair == dataMap().user_defined.end() )
        {
            plot_pair = dataMap().addUserDefined( prefixed_topic_name );
        }
        PlotDataAny& user_defined_data = plot_pair->second;
        user_defined_data.pushBack( PlotDataAny::Point(msg_time, nonstd::any(std::move(buffer)) ));
    }

    _ros_parser.extractData(dataMap(), _prefix);

    //------------------------------
    {
        int& index = _msg_index[topic_name];
        index++;
        const std::string key = prefixed_topic_name + ("/_MSG_INDEX_") ;
        auto index_it = dataMap().numeric.find(key);
        if( index_it == dataMap().numeric.end())
        {
            index_it = dataMap().addNumeric( key );
        }
        index_it->second.pushBack( PlotData::Point(msg_time, index) );
    }
}

void DataStreamROS::extractInitialSamples()
{
    using namespace std::chrono;
    milliseconds wait_time_ms(1000);

    QProgressDialog progress_dialog;
    progress_dialog.setLabelText( "Collecting ROS topic samples to understand data layout. ");
    progress_dialog.setRange(0, wait_time_ms.count());
    progress_dialog.setAutoClose(true);
    progress_dialog.setAutoReset(true);

    progress_dialog.show();

    auto start_time = system_clock::now();

    while ( system_clock::now() - start_time < (wait_time_ms) )
    {
        ros::getGlobalCallbackQueue()->callAvailable(ros::WallDuration(0.1));
        int i = duration_cast<milliseconds>(system_clock::now() - start_time).count() ;
        progress_dialog.setValue( i );
        QApplication::processEvents();
        if( progress_dialog.wasCanceled() )
        {
            break;
        }
    }

    if( progress_dialog.wasCanceled() == false )
    {
        progress_dialog.cancel();
    }
}

void DataStreamROS::timerCallback()
{
    if( _running && ros::master::check() == false
            && !_roscore_disconnection_already_notified)
    {
        auto ret = QMessageBox::warning(nullptr,
                                        tr("Disconnected!"),
                                        tr("The roscore master cannot be detected.\n\n"
                                           "Do you want to try reconnecting to it? \n\n"
                                           "NOTE: if you select CONTINUE, you might need"
                                           " to stop and restart this plugin."),
                                        tr("Stop Plugin"),
                                        tr("Try reconnect"),
                                        tr("Continue"),
                                        0);
        _roscore_disconnection_already_notified = ( ret == 2 );
        if( ret == 1 )
        {
            this->shutdown();
            _node =  RosManager::getNode();

            if( !_node ){
                emit connectionClosed();
                return;
            }
            subscribe();

            _running = true;
            _spinner = std::make_shared<ros::AsyncSpinner>(1);
            _spinner->start();
            _periodic_timer->start();
        }
        else if( ret == 0)
        {
            this->shutdown();
            emit connectionClosed();
        }
    }
}

void DataStreamROS::saveIntoRosbag(const PlotDataMapRef& data)
{

    if( data.user_defined.empty()){
        QMessageBox::warning(nullptr, tr("Warning"), tr("Your buffer is empty. Nothing to save.\n") );
        return;
    }

    QFileDialog saveDialog;
    saveDialog.setAcceptMode(QFileDialog::AcceptSave);
    saveDialog.setDefaultSuffix("bag");
    saveDialog.exec();

    if(saveDialog.result() != QDialog::Accepted || saveDialog.selectedFiles().empty())
    {
        return;
    }

    QString fileName = saveDialog.selectedFiles().first();

    if( fileName.size() > 0)
    {
        rosbag::Bag rosbag(fileName.toStdString(), rosbag::bagmode::Write );

        for (const auto& it: data.user_defined )
        {
            const std::string& topicname = it.first;
            const auto& plotdata = it.second;

            auto registered_msg_type = RosIntrospectionFactory::get().getShapeShifter(topicname);
            if(!registered_msg_type) continue;

            RosIntrospection::ShapeShifter msg;
            msg.morph(registered_msg_type->getMD5Sum(),
                      registered_msg_type->getDataType(),
                      registered_msg_type->getMessageDefinition());

            for (int i=0; i< plotdata.size(); i++)
            {
                const auto& point = plotdata.at(i);
                const PlotDataAny::TimeType msg_time  = point.x;
                const nonstd::any& type_erased_buffer = point.y;

                if(type_erased_buffer.type() != typeid( std::vector<uint8_t> ))
                {
                    // can't cast to expected type
                    continue;
                }

                std::vector<uint8_t> raw_buffer =  nonstd::any_cast<std::vector<uint8_t>>( type_erased_buffer );
                ros::serialization::IStream stream( raw_buffer.data(), raw_buffer.size() );
                msg.read( stream );

                rosbag.write( topicname, ros::Time(msg_time), msg);
            }
        }
        rosbag.close();

        QProcess process;
        QStringList args;
        args << "reindex" << fileName;
        process.start("rosbag" , args);
    }
}

void DataStreamROS::subscribe()
{
    _subscribers.clear();
    {
        boost::function<void(const rosgraph_msgs::Clock::ConstPtr&) > callback;
        callback = [this](const rosgraph_msgs::Clock::ConstPtr& msg) -> void
        {
            this->clockCallback(msg) ;
        };
    }

    for (int i=0; i< _config.selected_topics.size(); i++ )
    {
        const std::string topic_name = _config.selected_topics[i].toStdString();
        boost::function<void(const topic_tools::ShapeShifter::ConstPtr&) > callback;
        callback = [this, topic_name](const topic_tools::ShapeShifter::ConstPtr& msg) -> void
        {
            this->topicCallback(msg, topic_name) ;
        };

        ros::SubscribeOptions ops;
        ops.initByFullCallbackType(topic_name, 1, callback);
        ops.transport_hints = ros::TransportHints().tcpNoDelay();

        _subscribers.insert( {topic_name, _node->subscribe(ops) }  );
    }
}

bool DataStreamROS::start(QStringList* selected_datasources)
{
    _ros_parser.clear();
    if( !_node )
    {
        _node =  RosManager::getNode();
    }

    if( !_node ){
        return false;
    }

    {
        std::lock_guard<std::mutex> lock( mutex() );
        dataMap().numeric.clear();
        dataMap().user_defined.clear();
    }

    using namespace RosIntrospection;

    std::vector<std::pair<QString,QString>> all_topics;
    ros::master::V_TopicInfo topic_infos;
    ros::master::getTopics(topic_infos);
    for (ros::master::TopicInfo topic_info: topic_infos)
    {
        all_topics.push_back(
                    std::make_pair(QString(topic_info.name.c_str()),
                                   QString(topic_info.datatype.c_str()) ) );
    }

    QTimer timer;
    timer.setSingleShot(false);
    timer.setInterval( 1000);
    timer.start();

    DialogSelectRosTopics dialog( all_topics, _config );

    connect( &timer, &QTimer::timeout, [&]()
    {
        all_topics.clear();
        topic_infos.clear();
        ros::master::getTopics(topic_infos);
        for (ros::master::TopicInfo topic_info: topic_infos)
        {
            all_topics.push_back(
                        std::make_pair(QString(topic_info.name.c_str()),
                                       QString(topic_info.datatype.c_str()) ) );
        }
        dialog.updateTopicList(all_topics);
    });

    int res = dialog.exec();

    _config = dialog.getResult();

    timer.stop();

    if( res != QDialog::Accepted || _config.selected_topics.empty() )
    {
        return false;
    }

    saveDefaultSettings();

    _ros_parser.setUseHeaderStamp( _config.use_header_stamp );

    if( _config.use_renaming_rules )
    {
        _ros_parser.addRules( RuleEditing::getRenamingRules() );
    }

    _ros_parser.setMaxArrayPolicy( _config.max_array_size, _config.discard_large_arrays );

    //-------------------------
    subscribe();
    _running = true;

    extractInitialSamples();

    _spinner = std::make_shared<ros::AsyncSpinner>(1);
    _spinner->start();

    _periodic_timer->setInterval(500);
    _roscore_disconnection_already_notified = false;
    _periodic_timer->start();

    return true;
}

bool DataStreamROS::isRunning() const { return _running; }

void DataStreamROS::shutdown()
{
    _periodic_timer->stop();
    if(_spinner)
    {
        _spinner->stop();
    }
    for(auto& it: _subscribers)
    {
        it.second.shutdown();
    }
    _subscribers.clear();
    _running = false;
    _node.reset();
    _spinner.reset();
}

DataStreamROS::~DataStreamROS()
{
    shutdown();
}

bool DataStreamROS::xmlSaveState(QDomDocument &doc, QDomElement &plugin_elem) const
{
    QDomElement stamp_elem = doc.createElement("use_header_stamp");
    stamp_elem.setAttribute("value", _config.use_header_stamp ? "true" : "false");
    plugin_elem.appendChild( stamp_elem );

    QDomElement rename_elem = doc.createElement("use_renaming_rules");
    rename_elem.setAttribute("value", _config.use_renaming_rules ? "true" : "false");
    plugin_elem.appendChild( rename_elem );

    QDomElement discard_elem = doc.createElement("discard_large_arrays");
    discard_elem.setAttribute("value", _config.discard_large_arrays ? "true" : "false");
    plugin_elem.appendChild( discard_elem );

    QDomElement max_elem = doc.createElement("max_array_size");
    max_elem.setAttribute("value", QString::number(_config.max_array_size));
    plugin_elem.appendChild( max_elem );

    return true;
}

bool DataStreamROS::xmlLoadState(const QDomElement &parent_element)
{
    QDomElement stamp_elem = parent_element.firstChildElement( "use_header_stamp" );
    _config.use_header_stamp = ( stamp_elem.attribute("value") == "true");

    QDomElement rename_elem = parent_element.firstChildElement( "use_renaming_rules" );
    _config.use_renaming_rules = ( rename_elem.attribute("value") == "true");

    QDomElement discard_elem = parent_element.firstChildElement( "discard_large_arrays" );
    _config.discard_large_arrays = ( discard_elem.attribute("value") == "true");

    QDomElement max_elem = parent_element.firstChildElement( "max_array_size" );
    _config.max_array_size = max_elem.attribute("value").toInt();

    return true;
}

void DataStreamROS::addActionsToParentMenu(QMenu *menu)
{
    _action_saveIntoRosbag = new QAction(QString("Save cached value in a rosbag"), menu);
    menu->addAction( _action_saveIntoRosbag );

    connect( _action_saveIntoRosbag, &QAction::triggered, this, [this]()
    {
        DataStreamROS::saveIntoRosbag( *_destination_data );
    });
}

void DataStreamROS::saveDefaultSettings()
{
    QSettings settings;

    settings.setValue("DataStreamROS/default_topics", _config.selected_topics);
    settings.setValue("DataStreamROS/use_renaming", _config.use_renaming_rules);
    settings.setValue("DataStreamROS/use_header_stamp", _config.use_header_stamp);
    settings.setValue("DataStreamROS/max_array_size", (int)_config.max_array_size);
    settings.setValue("DataStreamROS/discard_large_arrays", _config.discard_large_arrays);
}

void DataStreamROS::loadDefaultSettings()
{
    QSettings settings;
    _config.selected_topics      = settings.value("DataStreamROS/default_topics", false ).toStringList();
    _config.use_header_stamp     = settings.value("DataStreamROS/use_header_stamp", false ).toBool();
    _config.use_renaming_rules   = settings.value("DataStreamROS/use_renaming", true ).toBool();
    _config.max_array_size       = settings.value("DataStreamROS/max_array_size", 100 ).toInt();
    _config.discard_large_arrays = settings.value("DataStreamROS/discard_large_arrays", true ).toBool();
}

<commit_msg>fix timestamp problem in streaming<commit_after>#include "datastream_ROS.h"
#include <QTextStream>
#include <QFile>
#include <QMessageBox>
#include <QDebug>
#include <thread>
#include <mutex>
#include <chrono>
#include <thread>
#include <QProgressDialog>
#include <QtGlobal>
#include <QApplication>
#include <QProcess>
#include <QCheckBox>
#include <QSettings>
#include <QFileDialog>
#include <ros/callback_queue.h>
#include <rosbag/bag.h>
#include <topic_tools/shape_shifter.h>
#include <ros/transport_hints.h>

#include "../dialog_select_ros_topics.h"
#include "../rule_editing.h"
#include "../qnodedialog.h"
#include "../shape_shifter_factory.hpp"

DataStreamROS::DataStreamROS():
    DataStreamer(),
    _node(nullptr),
    _destination_data(nullptr),
    _action_saveIntoRosbag(nullptr),
    _prev_clock_time(0)
{
    _running = false;
    _periodic_timer = new QTimer();
    connect( _periodic_timer, &QTimer::timeout,
             this, &DataStreamROS::timerCallback);

    loadDefaultSettings();
}

void DataStreamROS::topicCallback(const topic_tools::ShapeShifter::ConstPtr& msg,
                                  const std::string &topic_name)
{
    if( !_running ){
        return;
    }

    using namespace RosIntrospection;
    const auto&  md5sum     =  msg->getMD5Sum();
    const auto&  datatype   =  msg->getDataType();
    const auto&  definition =  msg->getMessageDefinition() ;

    // register the message type
    _ros_parser.registerSchema( topic_name, md5sum,
                                RosIntrospection::ROSType(datatype),
                                definition);

    RosIntrospectionFactory::registerMessage(topic_name, md5sum, datatype, definition );

    //------------------------------------

    // it is more efficient to recycle this elements
    static std::vector<uint8_t> buffer;

    buffer.resize( msg->size() );

    ros::serialization::OStream stream(buffer.data(), buffer.size());
    msg->write(stream);

    _ros_parser.setUseHeaderStamp( _config.use_header_stamp );

    double msg_time = ros::Time::now().toSec();
    if( msg_time == 0)
    {
      // corner case: use_sim_time == true but topic /clock is not published
      msg_time = ros::WallTime::now().toSec();
      _ros_parser.setUseHeaderStamp( false );
    }

    if( msg_time < _prev_clock_time )
    {
        // clear
        for (auto& it: dataMap().numeric ) {
            it.second.clear();
        }
        for (auto& it: dataMap().user_defined ){
            it.second.clear();
        }
        emit clearBuffers();
    }
    _prev_clock_time = msg_time;

    MessageRef buffer_view( buffer );
    _ros_parser.pushMessageRef( topic_name, buffer_view, msg_time );

    std::lock_guard<std::mutex> lock( mutex() );
    const std::string prefixed_topic_name = _prefix + topic_name;

    // adding raw serialized msg for future uses.
    // do this before msg_time normalization
    {
        auto plot_pair = dataMap().user_defined.find( prefixed_topic_name );
        if( plot_pair == dataMap().user_defined.end() )
        {
            plot_pair = dataMap().addUserDefined( prefixed_topic_name );
        }
        PlotDataAny& user_defined_data = plot_pair->second;
        user_defined_data.pushBack( PlotDataAny::Point(msg_time, nonstd::any(std::move(buffer)) ));
    }

    _ros_parser.extractData(dataMap(), _prefix);

    //------------------------------
    {
        int& index = _msg_index[topic_name];
        index++;
        const std::string key = prefixed_topic_name + ("/_MSG_INDEX_") ;
        auto index_it = dataMap().numeric.find(key);
        if( index_it == dataMap().numeric.end())
        {
            index_it = dataMap().addNumeric( key );
        }
        index_it->second.pushBack( PlotData::Point(msg_time, index) );
    }
}

void DataStreamROS::extractInitialSamples()
{
    using namespace std::chrono;
    milliseconds wait_time_ms(1000);

    QProgressDialog progress_dialog;
    progress_dialog.setLabelText( "Collecting ROS topic samples to understand data layout. ");
    progress_dialog.setRange(0, wait_time_ms.count());
    progress_dialog.setAutoClose(true);
    progress_dialog.setAutoReset(true);

    progress_dialog.show();

    auto start_time = system_clock::now();

    while ( system_clock::now() - start_time < (wait_time_ms) )
    {
        ros::getGlobalCallbackQueue()->callAvailable(ros::WallDuration(0.1));
        int i = duration_cast<milliseconds>(system_clock::now() - start_time).count() ;
        progress_dialog.setValue( i );
        QApplication::processEvents();
        if( progress_dialog.wasCanceled() )
        {
            break;
        }
    }

    if( progress_dialog.wasCanceled() == false )
    {
        progress_dialog.cancel();
    }
}

void DataStreamROS::timerCallback()
{
    if( _running && ros::master::check() == false
            && !_roscore_disconnection_already_notified)
    {
        auto ret = QMessageBox::warning(nullptr,
                                        tr("Disconnected!"),
                                        tr("The roscore master cannot be detected.\n\n"
                                           "Do you want to try reconnecting to it? \n\n"
                                           "NOTE: if you select CONTINUE, you might need"
                                           " to stop and restart this plugin."),
                                        tr("Stop Plugin"),
                                        tr("Try reconnect"),
                                        tr("Continue"),
                                        0);
        _roscore_disconnection_already_notified = ( ret == 2 );
        if( ret == 1 )
        {
            this->shutdown();
            _node =  RosManager::getNode();

            if( !_node ){
                emit connectionClosed();
                return;
            }
            subscribe();

            _running = true;
            _spinner = std::make_shared<ros::AsyncSpinner>(1);
            _spinner->start();
            _periodic_timer->start();
        }
        else if( ret == 0)
        {
            this->shutdown();
            emit connectionClosed();
        }
    }
}

void DataStreamROS::saveIntoRosbag(const PlotDataMapRef& data)
{

    if( data.user_defined.empty()){
        QMessageBox::warning(nullptr, tr("Warning"), tr("Your buffer is empty. Nothing to save.\n") );
        return;
    }

    QFileDialog saveDialog;
    saveDialog.setAcceptMode(QFileDialog::AcceptSave);
    saveDialog.setDefaultSuffix("bag");
    saveDialog.exec();

    if(saveDialog.result() != QDialog::Accepted || saveDialog.selectedFiles().empty())
    {
        return;
    }

    QString fileName = saveDialog.selectedFiles().first();

    if( fileName.size() > 0)
    {
        rosbag::Bag rosbag(fileName.toStdString(), rosbag::bagmode::Write );

        for (const auto& it: data.user_defined )
        {
            const std::string& topicname = it.first;
            const auto& plotdata = it.second;

            auto registered_msg_type = RosIntrospectionFactory::get().getShapeShifter(topicname);
            if(!registered_msg_type) continue;

            RosIntrospection::ShapeShifter msg;
            msg.morph(registered_msg_type->getMD5Sum(),
                      registered_msg_type->getDataType(),
                      registered_msg_type->getMessageDefinition());

            for (int i=0; i< plotdata.size(); i++)
            {
                const auto& point = plotdata.at(i);
                const PlotDataAny::TimeType msg_time  = point.x;
                const nonstd::any& type_erased_buffer = point.y;

                if(type_erased_buffer.type() != typeid( std::vector<uint8_t> ))
                {
                    // can't cast to expected type
                    continue;
                }

                std::vector<uint8_t> raw_buffer =  nonstd::any_cast<std::vector<uint8_t>>( type_erased_buffer );
                ros::serialization::IStream stream( raw_buffer.data(), raw_buffer.size() );
                msg.read( stream );

                rosbag.write( topicname, ros::Time(msg_time), msg);
            }
        }
        rosbag.close();

        QProcess process;
        QStringList args;
        args << "reindex" << fileName;
        process.start("rosbag" , args);
    }
}

void DataStreamROS::subscribe()
{
    _subscribers.clear();
    {
        boost::function<void(const rosgraph_msgs::Clock::ConstPtr&) > callback;
        callback = [this](const rosgraph_msgs::Clock::ConstPtr& msg) -> void
        {
            this->clockCallback(msg) ;
        };
    }

    for (int i=0; i< _config.selected_topics.size(); i++ )
    {
        const std::string topic_name = _config.selected_topics[i].toStdString();
        boost::function<void(const topic_tools::ShapeShifter::ConstPtr&) > callback;
        callback = [this, topic_name](const topic_tools::ShapeShifter::ConstPtr& msg) -> void
        {
            this->topicCallback(msg, topic_name) ;
        };

        ros::SubscribeOptions ops;
        ops.initByFullCallbackType(topic_name, 1, callback);
        ops.transport_hints = ros::TransportHints().tcpNoDelay();

        _subscribers.insert( {topic_name, _node->subscribe(ops) }  );
    }
}

bool DataStreamROS::start(QStringList* selected_datasources)
{
    _ros_parser.clear();
    if( !_node )
    {
        _node =  RosManager::getNode();
    }

    if( !_node ){
        return false;
    }

    {
        std::lock_guard<std::mutex> lock( mutex() );
        dataMap().numeric.clear();
        dataMap().user_defined.clear();
    }

    using namespace RosIntrospection;

    std::vector<std::pair<QString,QString>> all_topics;
    ros::master::V_TopicInfo topic_infos;
    ros::master::getTopics(topic_infos);
    for (ros::master::TopicInfo topic_info: topic_infos)
    {
        all_topics.push_back(
                    std::make_pair(QString(topic_info.name.c_str()),
                                   QString(topic_info.datatype.c_str()) ) );
    }

    QTimer timer;
    timer.setSingleShot(false);
    timer.setInterval( 1000);
    timer.start();

    DialogSelectRosTopics dialog( all_topics, _config );

    connect( &timer, &QTimer::timeout, [&]()
    {
        all_topics.clear();
        topic_infos.clear();
        ros::master::getTopics(topic_infos);
        for (ros::master::TopicInfo topic_info: topic_infos)
        {
            all_topics.push_back(
                        std::make_pair(QString(topic_info.name.c_str()),
                                       QString(topic_info.datatype.c_str()) ) );
        }
        dialog.updateTopicList(all_topics);
    });

    int res = dialog.exec();

    _config = dialog.getResult();

    timer.stop();

    if( res != QDialog::Accepted || _config.selected_topics.empty() )
    {
        return false;
    }

    saveDefaultSettings();

    if( _config.use_renaming_rules )
    {
        _ros_parser.addRules( RuleEditing::getRenamingRules() );
    }

    _ros_parser.setMaxArrayPolicy( _config.max_array_size, _config.discard_large_arrays );

    //-------------------------
    subscribe();
    _running = true;

    extractInitialSamples();

    _spinner = std::make_shared<ros::AsyncSpinner>(1);
    _spinner->start();

    _periodic_timer->setInterval(500);
    _roscore_disconnection_already_notified = false;
    _periodic_timer->start();

    return true;
}

bool DataStreamROS::isRunning() const { return _running; }

void DataStreamROS::shutdown()
{
    _periodic_timer->stop();
    if(_spinner)
    {
        _spinner->stop();
    }
    for(auto& it: _subscribers)
    {
        it.second.shutdown();
    }
    _subscribers.clear();
    _running = false;
    _node.reset();
    _spinner.reset();
}

DataStreamROS::~DataStreamROS()
{
    shutdown();
}

bool DataStreamROS::xmlSaveState(QDomDocument &doc, QDomElement &plugin_elem) const
{
    QDomElement stamp_elem = doc.createElement("use_header_stamp");
    stamp_elem.setAttribute("value", _config.use_header_stamp ? "true" : "false");
    plugin_elem.appendChild( stamp_elem );

    QDomElement rename_elem = doc.createElement("use_renaming_rules");
    rename_elem.setAttribute("value", _config.use_renaming_rules ? "true" : "false");
    plugin_elem.appendChild( rename_elem );

    QDomElement discard_elem = doc.createElement("discard_large_arrays");
    discard_elem.setAttribute("value", _config.discard_large_arrays ? "true" : "false");
    plugin_elem.appendChild( discard_elem );

    QDomElement max_elem = doc.createElement("max_array_size");
    max_elem.setAttribute("value", QString::number(_config.max_array_size));
    plugin_elem.appendChild( max_elem );

    return true;
}

bool DataStreamROS::xmlLoadState(const QDomElement &parent_element)
{
    QDomElement stamp_elem = parent_element.firstChildElement( "use_header_stamp" );
    _config.use_header_stamp = ( stamp_elem.attribute("value") == "true");

    QDomElement rename_elem = parent_element.firstChildElement( "use_renaming_rules" );
    _config.use_renaming_rules = ( rename_elem.attribute("value") == "true");

    QDomElement discard_elem = parent_element.firstChildElement( "discard_large_arrays" );
    _config.discard_large_arrays = ( discard_elem.attribute("value") == "true");

    QDomElement max_elem = parent_element.firstChildElement( "max_array_size" );
    _config.max_array_size = max_elem.attribute("value").toInt();

    return true;
}

void DataStreamROS::addActionsToParentMenu(QMenu *menu)
{
    _action_saveIntoRosbag = new QAction(QString("Save cached value in a rosbag"), menu);
    menu->addAction( _action_saveIntoRosbag );

    connect( _action_saveIntoRosbag, &QAction::triggered, this, [this]()
    {
        DataStreamROS::saveIntoRosbag( *_destination_data );
    });
}

void DataStreamROS::saveDefaultSettings()
{
    QSettings settings;

    settings.setValue("DataStreamROS/default_topics", _config.selected_topics);
    settings.setValue("DataStreamROS/use_renaming", _config.use_renaming_rules);
    settings.setValue("DataStreamROS/use_header_stamp", _config.use_header_stamp);
    settings.setValue("DataStreamROS/max_array_size", (int)_config.max_array_size);
    settings.setValue("DataStreamROS/discard_large_arrays", _config.discard_large_arrays);
}

void DataStreamROS::loadDefaultSettings()
{
    QSettings settings;
    _config.selected_topics      = settings.value("DataStreamROS/default_topics", false ).toStringList();
    _config.use_header_stamp     = settings.value("DataStreamROS/use_header_stamp", false ).toBool();
    _config.use_renaming_rules   = settings.value("DataStreamROS/use_renaming", true ).toBool();
    _config.max_array_size       = settings.value("DataStreamROS/max_array_size", 100 ).toInt();
    _config.discard_large_arrays = settings.value("DataStreamROS/discard_large_arrays", true ).toBool();
}

<|endoftext|>


<commit_before>#include "mesh/GPURenderTaskCollection.h"

namespace megamol {
namespace mesh {

void GPURenderTaskCollection::copyGPURenderTask(
    std::string const& identifier, RenderTaskMetaData render_task_meta_data) {
    //TODO!!!
}

void GPURenderTaskCollection::deleteRenderTask(std::string const& identifier) {

    glGetError(); //TODO fuck this

    auto query = m_render_task_meta_data.find(identifier);

    if (query != m_render_task_meta_data.end()) {

        auto rt_meta = query->second;

        auto rts = rt_meta.render_tasks;

        rts->draw_cnt -= 1;

        if (rts->draw_cnt == 0) {
            for (size_t i = 0; i < m_gpu_render_tasks.size(); ++i) {
                if (m_gpu_render_tasks[i] == rts) {
                    m_gpu_render_tasks.erase(m_gpu_render_tasks.begin() + i);
                    break;
                }
            }
        } else {

            // create new draw command and per draw data buffers with (size - 1)
            size_t old_dcs_byte_size = rts->draw_commands->getByteSize();
            size_t old_pdd_byte_size = rts->per_draw_data->getByteSize();
            size_t new_dcs_byte_size = old_dcs_byte_size - sizeof(glowl::DrawElementsCommand);
            size_t new_pdd_byte_size = old_pdd_byte_size - rt_meta.per_draw_data_byteSize;

            auto new_dcs_buffer = std::make_shared<glowl::BufferObject>(
                GL_DRAW_INDIRECT_BUFFER, nullptr, new_dcs_byte_size, GL_DYNAMIC_DRAW);
            auto new_pdd_buffer = std::make_shared<glowl::BufferObject>(
                GL_SHADER_STORAGE_BUFFER, nullptr, new_pdd_byte_size, GL_DYNAMIC_DRAW);

            // copy data from beg to delete
            glowl::BufferObject::copy(
                rts->draw_commands.get(), new_dcs_buffer.get(), 0, 0, rt_meta.draw_command_byteOffset);
            glowl::BufferObject::copy(
                rts->per_draw_data.get(), new_pdd_buffer.get(), 0, 0, rt_meta.per_draw_data_byteOffset);

            // copy data from delete to end
            glowl::BufferObject::copy(rts->draw_commands.get(), new_dcs_buffer.get(),
                rt_meta.draw_command_byteOffset + sizeof(glowl::DrawElementsCommand), rt_meta.draw_command_byteOffset,
                new_dcs_byte_size - rt_meta.draw_command_byteOffset);
            glowl::BufferObject::copy(rts->per_draw_data.get(), new_pdd_buffer.get(),
                rt_meta.per_draw_data_byteOffset + rt_meta.per_draw_data_byteSize, rt_meta.per_draw_data_byteOffset,
                new_pdd_byte_size - rt_meta.per_draw_data_byteOffset);

            rts->draw_commands = new_dcs_buffer;
            rts->per_draw_data = new_pdd_buffer;

        }

        m_render_task_meta_data.erase(query);
    }
}

void GPURenderTaskCollection::deletePerFrameDataBuffer(uint32_t buffer_binding_point) {
    for (int i = 0; i < m_per_frame_data_buffers.size(); ++i) {
        if (m_per_frame_data_buffers[i].second == buffer_binding_point) {
            m_per_frame_data_buffers.erase(m_per_frame_data_buffers.begin()+i);
        }
    }
}

void GPURenderTaskCollection::clear() {
    m_gpu_render_tasks.clear();
    m_per_frame_data_buffers.clear();
}

size_t GPURenderTaskCollection::getTotalDrawCount() {
    size_t retval = 0;
    for (auto& rt : m_gpu_render_tasks) {
        retval += rt->draw_cnt;
    }
    return retval;
};

std::vector<std::shared_ptr<GPURenderTaskCollection::GPURenderTask>> const& GPURenderTaskCollection::getRenderTasks() {
    return m_gpu_render_tasks;
}

GPURenderTaskCollection::RenderTaskMetaData const& GPURenderTaskCollection::getRenderTaskMetaData(
    std::string const& identifier) {
    RenderTaskMetaData retval;

    auto query = m_render_task_meta_data.find(identifier);

    if (query != m_render_task_meta_data.end()) {
        retval = query->second;
    } else {
        megamol::core::utility::log::Log::DefaultLog.WriteError("getRenderTaskMetaData error: identifier not found.");
    }

    return retval;
}

std::vector<std::pair<std::shared_ptr<glowl::BufferObject>, uint32_t>> const&
GPURenderTaskCollection::getPerFrameBuffers() {
    return m_per_frame_data_buffers;
}

void GPURenderTaskCollection::copyGPURenderTask(std::vector<GPURenderTask>& tgt_gpu_render_tasks,
    std::unordered_multimap<std::string, RenderTaskMetaData>& tgt_render_tas_meta_data, std::string const& identifier,
    RenderTaskMetaData const& src_render_task) {

}

} // namespace mesh
} // namespace megamol<commit_msg>Hotfix for render task collection. Won't wait for next master update...<commit_after>#include "mesh/GPURenderTaskCollection.h"

namespace megamol {
namespace mesh {

    void GPURenderTaskCollection::copyGPURenderTask(
        std::string const& identifier, RenderTaskMetaData render_task_meta_data) {
        // TODO!!!
    }

    void GPURenderTaskCollection::deleteRenderTask(std::string const& identifier) {

        glGetError(); // TODO fuck this

        auto query = m_render_task_meta_data.find(identifier);

        if (query != m_render_task_meta_data.end()) {

            auto rt_meta = query->second;

            auto rts = rt_meta.render_tasks;

            rts->draw_cnt -= 1;

            if (rts->draw_cnt == 0) {
                for (size_t i = 0; i < m_gpu_render_tasks.size(); ++i) {
                    if (m_gpu_render_tasks[i] == rts) {
                        m_gpu_render_tasks.erase(m_gpu_render_tasks.begin() + i);
                        break;
                    }
                }
            } else {

                // create new draw command and per draw data buffers with (size - 1)
                size_t old_dcs_byte_size = rts->draw_commands->getByteSize();
                size_t old_pdd_byte_size = rts->per_draw_data->getByteSize();
                size_t new_dcs_byte_size = old_dcs_byte_size - sizeof(glowl::DrawElementsCommand);
                size_t new_pdd_byte_size = old_pdd_byte_size - rt_meta.per_draw_data_byteSize;
                try {
                    auto new_dcs_buffer = std::make_shared<glowl::BufferObject>(
                        GL_DRAW_INDIRECT_BUFFER, nullptr, new_dcs_byte_size, GL_DYNAMIC_DRAW);
                    auto new_pdd_buffer = std::make_shared<glowl::BufferObject>(
                        GL_SHADER_STORAGE_BUFFER, nullptr, new_pdd_byte_size, GL_DYNAMIC_DRAW);

                    // copy data from beg to delete
                    glowl::BufferObject::copy(
                        rts->draw_commands.get(), new_dcs_buffer.get(), 0, 0, rt_meta.draw_command_byteOffset);
                    glowl::BufferObject::copy(
                        rts->per_draw_data.get(), new_pdd_buffer.get(), 0, 0, rt_meta.per_draw_data_byteOffset);

                    // copy data from delete to end
                    glowl::BufferObject::copy(rts->draw_commands.get(), new_dcs_buffer.get(),
                        rt_meta.draw_command_byteOffset + sizeof(glowl::DrawElementsCommand),
                        rt_meta.draw_command_byteOffset, new_dcs_byte_size - rt_meta.draw_command_byteOffset);
                    glowl::BufferObject::copy(rts->per_draw_data.get(), new_pdd_buffer.get(),
                        rt_meta.per_draw_data_byteOffset + rt_meta.per_draw_data_byteSize,
                        rt_meta.per_draw_data_byteOffset, new_pdd_byte_size - rt_meta.per_draw_data_byteOffset);

                    rts->draw_commands = new_dcs_buffer;
                    rts->per_draw_data = new_pdd_buffer;

                    // TODO!!!! iterate all meta data entries: do nothing if byte offset < delete; decrement if byte
                    // offset > delete
                    for (auto& rtm : m_render_task_meta_data) {
                        if (rtm.second.draw_command_byteOffset > rt_meta.draw_command_byteOffset) {
                            rtm.second.draw_command_byteOffset -= sizeof(glowl::DrawElementsCommand);
                        }
                        if (rtm.second.per_draw_data_byteOffset > rt_meta.per_draw_data_byteOffset) {
                            rtm.second.per_draw_data_byteOffset -= rt_meta.per_draw_data_byteSize;
                        }
                    }

                } catch (glowl::BufferObjectException const& e) {
                    megamol::core::utility::log::Log::DefaultLog.WriteMsg(megamol::core::utility::log::Log::LEVEL_ERROR,
                        "Error during GPU render task deletion: %s\n", e.what());
                }
            }

            m_render_task_meta_data.erase(query);
        }
    }

    void GPURenderTaskCollection::deletePerFrameDataBuffer(uint32_t buffer_binding_point) {
        for (int i = 0; i < m_per_frame_data_buffers.size(); ++i) {
            if (m_per_frame_data_buffers[i].second == buffer_binding_point) {
                m_per_frame_data_buffers.erase(m_per_frame_data_buffers.begin() + i);
            }
        }
    }

    void GPURenderTaskCollection::clear() {
        m_gpu_render_tasks.clear();
        m_per_frame_data_buffers.clear();
    }

    size_t GPURenderTaskCollection::getTotalDrawCount() {
        size_t retval = 0;
        for (auto& rt : m_gpu_render_tasks) {
            retval += rt->draw_cnt;
        }
        return retval;
    };

    std::vector<std::shared_ptr<GPURenderTaskCollection::GPURenderTask>> const&
    GPURenderTaskCollection::getRenderTasks() {
        return m_gpu_render_tasks;
    }

    GPURenderTaskCollection::RenderTaskMetaData const& GPURenderTaskCollection::getRenderTaskMetaData(
        std::string const& identifier) {
        RenderTaskMetaData retval;

        auto query = m_render_task_meta_data.find(identifier);

        if (query != m_render_task_meta_data.end()) {
            retval = query->second;
        } else {
            megamol::core::utility::log::Log::DefaultLog.WriteError(
                "getRenderTaskMetaData error: identifier not found.");
        }

        return retval;
    }

    std::vector<std::pair<std::shared_ptr<glowl::BufferObject>, uint32_t>> const&
    GPURenderTaskCollection::getPerFrameBuffers() {
        return m_per_frame_data_buffers;
    }

    void GPURenderTaskCollection::copyGPURenderTask(std::vector<GPURenderTask>& tgt_gpu_render_tasks,
        std::unordered_multimap<std::string, RenderTaskMetaData>& tgt_render_tas_meta_data,
        std::string const& identifier, RenderTaskMetaData const& src_render_task) {}

} // namespace mesh
} // namespace megamol
<|endoftext|>


<commit_before>//===--- Compiler.cpp --------------------------------------------*- C++-*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "Compiler.h"
#include "Logger.h"
#include "clang/Basic/TargetInfo.h"
#include "clang/Lex/PreprocessorOptions.h"
#include "llvm/Support/Format.h"
#include "llvm/Support/FormatVariadic.h"

namespace clang {
namespace clangd {

void IgnoreDiagnostics::log(DiagnosticsEngine::Level DiagLevel,
                            const clang::Diagnostic &Info) {
  // FIXME: format lazily, in case vlog is off.
  llvm::SmallString<64> Message;
  Info.FormatDiagnostic(Message);

  llvm::SmallString<64> Location;
  if (Info.hasSourceManager() && Info.getLocation().isValid()) {
    auto &SourceMgr = Info.getSourceManager();
    auto Loc = SourceMgr.getFileLoc(Info.getLocation());
    llvm::raw_svector_ostream OS(Location);
    Loc.print(OS, SourceMgr);
    OS << ":";
  }

  clangd::vlog("Ignored diagnostic. {0}{1}", Location, Message);
}

void IgnoreDiagnostics::HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
                                         const clang::Diagnostic &Info) {
  IgnoreDiagnostics::log(DiagLevel, Info);
}

std::unique_ptr<CompilerInvocation>
buildCompilerInvocation(const ParseInputs &Inputs) {
  std::vector<const char *> ArgStrs;
  for (const auto &S : Inputs.CompileCommand.CommandLine)
    ArgStrs.push_back(S.c_str());

  if (Inputs.FS->setCurrentWorkingDirectory(Inputs.CompileCommand.Directory)) {
    log("Couldn't set working directory when creating compiler invocation.");
    // We proceed anyway, our lit-tests rely on results for non-existing working
    // dirs.
  }

  // FIXME(ibiryukov): store diagnostics from CommandLine when we start
  // reporting them.
  IgnoreDiagnostics IgnoreDiagnostics;
  llvm::IntrusiveRefCntPtr<DiagnosticsEngine> CommandLineDiagsEngine =
      CompilerInstance::createDiagnostics(new DiagnosticOptions,
                                          &IgnoreDiagnostics, false);
  std::unique_ptr<CompilerInvocation> CI = createInvocationFromCommandLine(
      ArgStrs, CommandLineDiagsEngine, Inputs.FS);
  if (!CI)
    return nullptr;
  // createInvocationFromCommandLine sets DisableFree.
  CI->getFrontendOpts().DisableFree = false;
  CI->getLangOpts()->CommentOpts.ParseAllComments = true;
  return CI;
}

std::unique_ptr<CompilerInstance>
prepareCompilerInstance(std::unique_ptr<clang::CompilerInvocation> CI,
                        const PrecompiledPreamble *Preamble,
                        std::unique_ptr<llvm::MemoryBuffer> Buffer,
                        std::shared_ptr<PCHContainerOperations> PCHs,
                        llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
                        DiagnosticConsumer &DiagsClient) {
  assert(VFS && "VFS is null");
  assert(!CI->getPreprocessorOpts().RetainRemappedFileBuffers &&
         "Setting RetainRemappedFileBuffers to true will cause a memory leak "
         "of ContentsBuffer");

  // NOTE: we use Buffer.get() when adding remapped files, so we have to make
  // sure it will be released if no error is emitted.
  if (Preamble) {
    Preamble->OverridePreamble(*CI, VFS, Buffer.get());
  } else {
    CI->getPreprocessorOpts().addRemappedFile(
        CI->getFrontendOpts().Inputs[0].getFile(), Buffer.get());
  }

  auto Clang = llvm::make_unique<CompilerInstance>(PCHs);
  Clang->setInvocation(std::move(CI));
  Clang->createDiagnostics(&DiagsClient, false);

  if (auto VFSWithRemapping = createVFSFromCompilerInvocation(
          Clang->getInvocation(), Clang->getDiagnostics(), VFS))
    VFS = VFSWithRemapping;
  Clang->setVirtualFileSystem(VFS);

  Clang->setTarget(TargetInfo::CreateTargetInfo(
      Clang->getDiagnostics(), Clang->getInvocation().TargetOpts));
  if (!Clang->hasTarget())
    return nullptr;

  // RemappedFileBuffers will handle the lifetime of the Buffer pointer,
  // release it.
  Buffer.release();
  return Clang;
}

} // namespace clangd
} // namespace clang
<commit_msg>Frontend: Remove CompilerInstance::VirtualFileSystem, NFC<commit_after>//===--- Compiler.cpp --------------------------------------------*- C++-*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "Compiler.h"
#include "Logger.h"
#include "clang/Basic/TargetInfo.h"
#include "clang/Lex/PreprocessorOptions.h"
#include "llvm/Support/Format.h"
#include "llvm/Support/FormatVariadic.h"

namespace clang {
namespace clangd {

void IgnoreDiagnostics::log(DiagnosticsEngine::Level DiagLevel,
                            const clang::Diagnostic &Info) {
  // FIXME: format lazily, in case vlog is off.
  llvm::SmallString<64> Message;
  Info.FormatDiagnostic(Message);

  llvm::SmallString<64> Location;
  if (Info.hasSourceManager() && Info.getLocation().isValid()) {
    auto &SourceMgr = Info.getSourceManager();
    auto Loc = SourceMgr.getFileLoc(Info.getLocation());
    llvm::raw_svector_ostream OS(Location);
    Loc.print(OS, SourceMgr);
    OS << ":";
  }

  clangd::vlog("Ignored diagnostic. {0}{1}", Location, Message);
}

void IgnoreDiagnostics::HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
                                         const clang::Diagnostic &Info) {
  IgnoreDiagnostics::log(DiagLevel, Info);
}

std::unique_ptr<CompilerInvocation>
buildCompilerInvocation(const ParseInputs &Inputs) {
  std::vector<const char *> ArgStrs;
  for (const auto &S : Inputs.CompileCommand.CommandLine)
    ArgStrs.push_back(S.c_str());

  if (Inputs.FS->setCurrentWorkingDirectory(Inputs.CompileCommand.Directory)) {
    log("Couldn't set working directory when creating compiler invocation.");
    // We proceed anyway, our lit-tests rely on results for non-existing working
    // dirs.
  }

  // FIXME(ibiryukov): store diagnostics from CommandLine when we start
  // reporting them.
  IgnoreDiagnostics IgnoreDiagnostics;
  llvm::IntrusiveRefCntPtr<DiagnosticsEngine> CommandLineDiagsEngine =
      CompilerInstance::createDiagnostics(new DiagnosticOptions,
                                          &IgnoreDiagnostics, false);
  std::unique_ptr<CompilerInvocation> CI = createInvocationFromCommandLine(
      ArgStrs, CommandLineDiagsEngine, Inputs.FS);
  if (!CI)
    return nullptr;
  // createInvocationFromCommandLine sets DisableFree.
  CI->getFrontendOpts().DisableFree = false;
  CI->getLangOpts()->CommentOpts.ParseAllComments = true;
  return CI;
}

std::unique_ptr<CompilerInstance>
prepareCompilerInstance(std::unique_ptr<clang::CompilerInvocation> CI,
                        const PrecompiledPreamble *Preamble,
                        std::unique_ptr<llvm::MemoryBuffer> Buffer,
                        std::shared_ptr<PCHContainerOperations> PCHs,
                        llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
                        DiagnosticConsumer &DiagsClient) {
  assert(VFS && "VFS is null");
  assert(!CI->getPreprocessorOpts().RetainRemappedFileBuffers &&
         "Setting RetainRemappedFileBuffers to true will cause a memory leak "
         "of ContentsBuffer");

  // NOTE: we use Buffer.get() when adding remapped files, so we have to make
  // sure it will be released if no error is emitted.
  if (Preamble) {
    Preamble->OverridePreamble(*CI, VFS, Buffer.get());
  } else {
    CI->getPreprocessorOpts().addRemappedFile(
        CI->getFrontendOpts().Inputs[0].getFile(), Buffer.get());
  }

  auto Clang = llvm::make_unique<CompilerInstance>(PCHs);
  Clang->setInvocation(std::move(CI));
  Clang->createDiagnostics(&DiagsClient, false);

  if (auto VFSWithRemapping = createVFSFromCompilerInvocation(
          Clang->getInvocation(), Clang->getDiagnostics(), VFS))
    VFS = VFSWithRemapping;
  Clang->createFileManager(VFS);

  Clang->setTarget(TargetInfo::CreateTargetInfo(
      Clang->getDiagnostics(), Clang->getInvocation().TargetOpts));
  if (!Clang->hasTarget())
    return nullptr;

  // RemappedFileBuffers will handle the lifetime of the Buffer pointer,
  // release it.
  Buffer.release();
  return Clang;
}

} // namespace clangd
} // namespace clang
<|endoftext|>