<commit_before>/*===================================================================

The Medical Imaging Interaction Toolkit (MITK)

Copyright (c) German Cancer Research Center,
Division of Medical and Biological Informatics.
All rights reserved.

This software is distributed WITHOUT ANY WARRANTY; without
even the implied warranty of MERCHANTABILITY or FITNESS FOR
A PARTICULAR PURPOSE.

See LICENSE.txt or http://www.mitk.org for details.

===================================================================*/

#include "QmitkPythonVariableStackTableModel.h"
#include <QMimeData>
#include <usModuleContext.h>
#include <mitkDataNode.h>
#include <usGetModuleContext.h>
#include <QStringList>
#include <QMessageBox>

const QString QmitkPythonVariableStackTableModel::MITK_IMAGE_VAR_NAME = "mitkImage";
const QString QmitkPythonVariableStackTableModel::MITK_SURFACE_VAR_NAME = "mitkSurface";

QmitkPythonVariableStackTableModel::QmitkPythonVariableStackTableModel(QObject *parent)
    :QAbstractTableModel(parent)
{
    us::ModuleContext* context = us::GetModuleContext();
    m_PythonServiceRef = context->GetServiceReference<mitk::IPythonService>();
    m_PythonService = context->GetService<mitk::IPythonService>(m_PythonServiceRef);
    m_PythonService->AddPythonCommandObserver( this );
}

QmitkPythonVariableStackTableModel::~QmitkPythonVariableStackTableModel()
{
  us::ModuleContext* context = us::GetModuleContext();
  context->UngetService( m_PythonServiceRef );
  m_PythonService->RemovePythonCommandObserver( this );
}

bool QmitkPythonVariableStackTableModel::dropMimeData ( const QMimeData * data, Qt::DropAction action, int row, int column, const QModelIndex & parent )
{
    // Early exit, returning true, but not actually doing anything (ignoring data).
    if (action == Qt::IgnoreAction)
        return true;

    // Note, we are returning true if we handled it, and false otherwise
    bool returnValue = false;

    if(data->hasFormat("application/x-mitk-datanodes"))
    {
        MITK_DEBUG("QmitkPythonVariableStackTableModel") << "dropped MITK DataNode";
        returnValue = true;

        QString arg = QString(data->data("application/x-mitk-datanodes").data());
        QStringList listOfDataNodeAddressPointers = arg.split(",");

        QStringList::iterator slIter;
        int i = 0;
        int j = 0;
        for (slIter = listOfDataNodeAddressPointers.begin();
             slIter != listOfDataNodeAddressPointers.end();
             slIter++)
        {
          long val = (*slIter).toLong();
          mitk::DataNode* node = static_cast<mitk::DataNode *>((void*)val);
          mitk::Image* mitkImage = dynamic_cast<mitk::Image*>(node->GetData());
          MITK_DEBUG("QmitkPythonVariableStackTableModel") << "mitkImage is not null " << (mitkImage != 0? "true": "false");

          if( mitkImage )
          {
            QString varName = MITK_IMAGE_VAR_NAME;
            if( i > 0 )
              varName = QString("%1%2").arg(MITK_IMAGE_VAR_NAME).arg(i);
            MITK_DEBUG("QmitkPythonVariableStackTableModel") << "varName" << varName.toStdString();

            bool exportAsCvImage = mitkImage->GetDimension() == 2 && m_PythonService->IsOpenCvPythonWrappingAvailable();

            if( exportAsCvImage )
            {
              int ret = QMessageBox::question(NULL, "Export option",
                "2D image detected. Export as OpenCV image to Python instead of an ITK image?",
                                              QMessageBox::Yes | QMessageBox::No, QMessageBox::No);

              exportAsCvImage = ret == QMessageBox::Yes;
              if(exportAsCvImage)
              {
                m_PythonService->CopyToPythonAsCvImage( mitkImage, MITK_IMAGE_VAR_NAME.toStdString() );
                ++i;
              }
            }
            if( !exportAsCvImage )
            {
              if( m_PythonService->IsItkPythonWrappingAvailable() )
              {
                m_PythonService->CopyToPythonAsItkImage( mitkImage, MITK_IMAGE_VAR_NAME.toStdString() );
                ++i;
              }
              else
              {
                MITK_ERROR << "ITK Python wrapping not available. Skipping export for image " << node->GetName();
              }
            }
          }
          else
          {
            mitk::Surface* surface = dynamic_cast<mitk::Surface*>(node->GetData());
            MITK_DEBUG("QmitkPythonVariableStackTableModel") << "found surface";

            if( surface )
            {
              QString varName = MITK_SURFACE_VAR_NAME;
              MITK_DEBUG("QmitkPythonVariableStackTableModel") << "varName" << varName.toStdString();

              if( j > 0 )
                varName = QString("%1%2").arg(MITK_SURFACE_VAR_NAME).arg(j);
              MITK_DEBUG("varName") << "varName" << varName.toStdString();

              if( m_PythonService->IsVtkPythonWrappingAvailable() )
              {
                m_PythonService->CopyToPythonAsVtkPolyData( surface, MITK_SURFACE_VAR_NAME.toStdString() );
                ++j;
              }
              else
              {
                MITK_ERROR << "VTK Python wrapping not available. Skipping export for surface " << node->GetName();
              }
            }
          }
        }
    }
    return returnValue;
}

QVariant QmitkPythonVariableStackTableModel::headerData(int section, Qt::Orientation orientation,
                                                        int role) const
{
    QVariant headerData;

    // show only horizontal header
    if ( role == Qt::DisplayRole )
    {
        if( orientation == Qt::Horizontal )
        {
            // first column: "Attribute"
            if(section == 0)
                headerData = "Attribute";
            else if(section == 1)
                headerData = "Type";
            else if(section == 2)
                headerData = "Value";
        }
    }

    return headerData;
}

Qt::ItemFlags QmitkPythonVariableStackTableModel::flags(const QModelIndex &index) const
{
    Qt::ItemFlags flags = QAbstractItemModel::flags(index);

    if(index.isValid())
        return Qt::ItemIsDragEnabled | Qt::ItemIsDropEnabled | flags;
    else
        return Qt::ItemIsDropEnabled | flags;
}

int QmitkPythonVariableStackTableModel::rowCount(const QModelIndex &) const
{
    return m_VariableStack.size();
}

int QmitkPythonVariableStackTableModel::columnCount(const QModelIndex &) const
{
    return 3;
}

QVariant QmitkPythonVariableStackTableModel::data(const QModelIndex &index, int role) const
{
    if (index.isValid() && !m_VariableStack.empty())
    {
        if(role == Qt::DisplayRole)
        {
            mitk::PythonVariable item = m_VariableStack.at(index.row());
            if(index.column() == 0)
              return QString::fromStdString(item.m_Name);
            if(index.column() == 1)
                return QString::fromStdString(item.m_Type);
            if(index.column() == 2)
                return QString::fromStdString(item.m_Value);
        }
    }
    return QVariant();
}

QStringList QmitkPythonVariableStackTableModel::mimeTypes() const
{
    return QAbstractTableModel::mimeTypes();
    QStringList types;
    types << "application/x-mitk-datanodes";
    types << "application/x-qabstractitemmodeldatalist";
    return types;
}

Qt::DropActions QmitkPythonVariableStackTableModel::supportedDropActions() const
{
    return Qt::CopyAction | Qt::MoveAction;
}

void QmitkPythonVariableStackTableModel::CommandExecuted(const std::string& pythonCommand)
{
  MITK_DEBUG("QmitkPythonVariableStackTableModel") << "command was executed " << pythonCommand;
    m_VariableStack = m_PythonService->GetVariableStack();
    QAbstractTableModel::reset();
}

std::vector<mitk::PythonVariable> QmitkPythonVariableStackTableModel::GetVariableStack() const
{
    return m_VariableStack;
}
<commit_msg>adding nodes with names other than mitkSurface to python<commit_after>/*===================================================================

The Medical Imaging Interaction Toolkit (MITK)

Copyright (c) German Cancer Research Center,
Division of Medical and Biological Informatics.
All rights reserved.

This software is distributed WITHOUT ANY WARRANTY; without
even the implied warranty of MERCHANTABILITY or FITNESS FOR
A PARTICULAR PURPOSE.

See LICENSE.txt or http://www.mitk.org for details.

===================================================================*/

#include "QmitkPythonVariableStackTableModel.h"
#include <QMimeData>
#include <usModuleContext.h>
#include <mitkDataNode.h>
#include <usGetModuleContext.h>
#include <QStringList>
#include <QMessageBox>

const QString QmitkPythonVariableStackTableModel::MITK_IMAGE_VAR_NAME = "mitkImage";
const QString QmitkPythonVariableStackTableModel::MITK_SURFACE_VAR_NAME = "mitkSurface";

QmitkPythonVariableStackTableModel::QmitkPythonVariableStackTableModel(QObject *parent)
    :QAbstractTableModel(parent)
{
    us::ModuleContext* context = us::GetModuleContext();
    m_PythonServiceRef = context->GetServiceReference<mitk::IPythonService>();
    m_PythonService = context->GetService<mitk::IPythonService>(m_PythonServiceRef);
    m_PythonService->AddPythonCommandObserver( this );
}

QmitkPythonVariableStackTableModel::~QmitkPythonVariableStackTableModel()
{
  us::ModuleContext* context = us::GetModuleContext();
  context->UngetService( m_PythonServiceRef );
  m_PythonService->RemovePythonCommandObserver( this );
}

bool QmitkPythonVariableStackTableModel::dropMimeData ( const QMimeData * data, Qt::DropAction action, int row, int column, const QModelIndex & parent )
{
    // Early exit, returning true, but not actually doing anything (ignoring data).
    if (action == Qt::IgnoreAction)
        return true;

    // Note, we are returning true if we handled it, and false otherwise
    bool returnValue = false;

    if(data->hasFormat("application/x-mitk-datanodes"))
    {
        MITK_DEBUG("QmitkPythonVariableStackTableModel") << "dropped MITK DataNode";
        returnValue = true;

        QString arg = QString(data->data("application/x-mitk-datanodes").data());
        QStringList listOfDataNodeAddressPointers = arg.split(",");

        QStringList::iterator slIter;
        int i = 0;
        int j = 0;
        for (slIter = listOfDataNodeAddressPointers.begin();
             slIter != listOfDataNodeAddressPointers.end();
             slIter++)
        {
          long val = (*slIter).toLong();
          mitk::DataNode* node = static_cast<mitk::DataNode *>((void*)val);
          mitk::Image* mitkImage = dynamic_cast<mitk::Image*>(node->GetData());
          MITK_DEBUG("QmitkPythonVariableStackTableModel") << "mitkImage is not null " << (mitkImage != 0? "true": "false");

          if( mitkImage )
          {
            QString varName = MITK_IMAGE_VAR_NAME;
            if( i > 0 )
              varName = QString("%1%2").arg(MITK_IMAGE_VAR_NAME).arg(i);
            MITK_DEBUG("QmitkPythonVariableStackTableModel") << "varName" << varName.toStdString();

            bool exportAsCvImage = mitkImage->GetDimension() == 2 && m_PythonService->IsOpenCvPythonWrappingAvailable();

            if( exportAsCvImage )
            {
              int ret = QMessageBox::question(NULL, "Export option",
                "2D image detected. Export as OpenCV image to Python instead of an ITK image?",
                                              QMessageBox::Yes | QMessageBox::No, QMessageBox::No);

              exportAsCvImage = ret == QMessageBox::Yes;
              if(exportAsCvImage)
              {
                m_PythonService->CopyToPythonAsCvImage( mitkImage, MITK_IMAGE_VAR_NAME.toStdString() );
                ++i;
              }
            }
            if( !exportAsCvImage )
            {
              if( m_PythonService->IsItkPythonWrappingAvailable() )
              {
                m_PythonService->CopyToPythonAsItkImage( mitkImage, MITK_IMAGE_VAR_NAME.toStdString() );
                ++i;
              }
              else
              {
                MITK_ERROR << "ITK Python wrapping not available. Skipping export for image " << node->GetName();
              }
            }
          }
          else
          {
            mitk::Surface* surface = dynamic_cast<mitk::Surface*>(node->GetData());
            MITK_DEBUG("QmitkPythonVariableStackTableModel") << "found surface";

            if( surface )
            {
              MITK_DEBUG("QmitkPythonVariableStackTableModel") << "varName" << node->GetName();

              if( m_PythonService->IsVtkPythonWrappingAvailable() )
              {
                m_PythonService->CopyToPythonAsVtkPolyData( surface, node->GetName() );
              }
              else
              {
                MITK_ERROR << "VTK Python wrapping not available. Skipping export for surface " << node->GetName();
              }
            }
          }
        }
    }
    return returnValue;
}

QVariant QmitkPythonVariableStackTableModel::headerData(int section, Qt::Orientation orientation,
                                                        int role) const
{
    QVariant headerData;

    // show only horizontal header
    if ( role == Qt::DisplayRole )
    {
        if( orientation == Qt::Horizontal )
        {
            // first column: "Attribute"
            if(section == 0)
                headerData = "Attribute";
            else if(section == 1)
                headerData = "Type";
            else if(section == 2)
                headerData = "Value";
        }
    }

    return headerData;
}

Qt::ItemFlags QmitkPythonVariableStackTableModel::flags(const QModelIndex &index) const
{
    Qt::ItemFlags flags = QAbstractItemModel::flags(index);

    if(index.isValid())
        return Qt::ItemIsDragEnabled | Qt::ItemIsDropEnabled | flags;
    else
        return Qt::ItemIsDropEnabled | flags;
}

int QmitkPythonVariableStackTableModel::rowCount(const QModelIndex &) const
{
    return m_VariableStack.size();
}

int QmitkPythonVariableStackTableModel::columnCount(const QModelIndex &) const
{
    return 3;
}

QVariant QmitkPythonVariableStackTableModel::data(const QModelIndex &index, int role) const
{
    if (index.isValid() && !m_VariableStack.empty())
    {
        if(role == Qt::DisplayRole)
        {
            mitk::PythonVariable item = m_VariableStack.at(index.row());
            if(index.column() == 0)
              return QString::fromStdString(item.m_Name);
            if(index.column() == 1)
                return QString::fromStdString(item.m_Type);
            if(index.column() == 2)
                return QString::fromStdString(item.m_Value);
        }
    }
    return QVariant();
}

QStringList QmitkPythonVariableStackTableModel::mimeTypes() const
{
    return QAbstractTableModel::mimeTypes();
    QStringList types;
    types << "application/x-mitk-datanodes";
    types << "application/x-qabstractitemmodeldatalist";
    return types;
}

Qt::DropActions QmitkPythonVariableStackTableModel::supportedDropActions() const
{
    return Qt::CopyAction | Qt::MoveAction;
}

void QmitkPythonVariableStackTableModel::CommandExecuted(const std::string& pythonCommand)
{
  MITK_DEBUG("QmitkPythonVariableStackTableModel") << "command was executed " << pythonCommand;
    m_VariableStack = m_PythonService->GetVariableStack();
    QAbstractTableModel::reset();
}

std::vector<mitk::PythonVariable> QmitkPythonVariableStackTableModel::GetVariableStack() const
{
    return m_VariableStack;
}
<|endoftext|>


<commit_before>/*===================================================================

The Medical Imaging Interaction Toolkit (MITK)

Copyright (c) German Cancer Research Center,
Division of Medical and Biological Informatics.
All rights reserved.

This software is distributed WITHOUT ANY WARRANTY; without
even the implied warranty of MERCHANTABILITY or FITNESS FOR
A PARTICULAR PURPOSE.

See LICENSE.txt or http://www.mitk.org for details.

===================================================================*/
#include <mitkTestingMacros.h>
#include <mitkContourModel.h>

//Add a vertex to the contour and see if size changed
static void TestAddVertex()
{
  mitk::ContourModel::Pointer contour = mitk::ContourModel::New();

  mitk::Point3D point;
  point[0] = point[1] = point[2] = 0;

  contour->AddVertex(point);

  MITK_TEST_CONDITION(contour->GetNumberOfVertices() > 0, "Add a Vertex, size increased");

}

//select a vertex by index. successful if the selected vertex member of the contour is no longer set to null
static void TestSelectVertexAtIndex()
{
  mitk::ContourModel::Pointer contour = mitk::ContourModel::New();

  mitk::Point3D point;
  point[0] = point[1] = point[2] = 0;

  contour->AddVertex(point);

  contour->SelectVertexAt(0);

  MITK_TEST_CONDITION(contour->GetSelectedVertex() != NULL, "Vertex was selected at index");
}

//select a vertex by worldposition. successful if the selected vertex member of the contour is no longer set to null
static void TestSelectVertexAtWorldposition()
{
  mitk::ContourModel::Pointer contour = mitk::ContourModel::New();

  mitk::Point3D point;
  point[0] = point[1] = point[2] = 0;

  contour->AddVertex(point);

  //same point is used here so the epsilon can be chosen very small
  contour->SelectVertexAt(point, 0.01);

  MITK_TEST_CONDITION(contour->GetSelectedVertex() != NULL, "Vertex was selected at index");
}

//move a vertex by a translation vector
static void TestMoveSelectedVertex()
{
  mitk::ContourModel::Pointer contour = mitk::ContourModel::New();

  mitk::Point3D point;
  point[0] = point[1] = point[2] = 0;

  contour->AddVertex(point);

  //same point is used here so the epsilon can be chosen very small
  contour->SelectVertexAt(point, 0.01);

  mitk::Vector3D v;
  v[0] = 1;
  v[1] = 3;
  v[2] = -1;

  contour->MoveSelectedVertex(v);

  const mitk::ContourModel::VertexType* vertex = contour->GetSelectedVertex();

  bool correctlyMoved = false;

  correctlyMoved = (*(vertex->Coordinates))[0] == (v[0]) &&
                   (*(vertex->Coordinates))[1] == (v[1]) &&
                   (*(vertex->Coordinates))[2] == (v[2]);

  MITK_TEST_CONDITION(correctlyMoved, "Vertex has been moved");
}

int mitkContourModelTest(int argc, char* argv[])
{
  MITK_TEST_BEGIN("mitkContourModelTest")

    TestAddVertex();
  TestSelectVertexAtIndex();
  TestSelectVertexAtWorldposition();
  MITK_TEST_END()
}<commit_msg>test methods for contourModel<commit_after>/*===================================================================

The Medical Imaging Interaction Toolkit (MITK)

Copyright (c) German Cancer Research Center,
Division of Medical and Biological Informatics.
All rights reserved.

This software is distributed WITHOUT ANY WARRANTY; without
even the implied warranty of MERCHANTABILITY or FITNESS FOR
A PARTICULAR PURPOSE.

See LICENSE.txt or http://www.mitk.org for details.

===================================================================*/
#include <mitkTestingMacros.h>
#include <mitkContourModel.h>

//Add a vertex to the contour and see if size changed
static void TestAddVertex()
{
  mitk::ContourModel::Pointer contour = mitk::ContourModel::New();

  mitk::Point3D p;
  p[0] = p[1] = p[2] = 0;

  contour->AddVertex(p);

  MITK_TEST_CONDITION(contour->GetNumberOfVertices() > 0, "Add a Vertex, size increased");

}

//select a vertex by index. successful if the selected vertex member of the contour is no longer set to null
static void TestSelectVertexAtIndex()
{
  mitk::ContourModel::Pointer contour = mitk::ContourModel::New();

  mitk::Point3D p;
  p[0] = p[1] = p[2] = 0;

  contour->AddVertex(p);

  contour->SelectVertexAt(0);

  MITK_TEST_CONDITION(contour->GetSelectedVertex() != NULL, "Vertex was selected at index");
}

//select a vertex by worldposition. successful if the selected vertex member of the contour is no longer set to null
static void TestSelectVertexAtWorldposition()
{
  mitk::ContourModel::Pointer contour = mitk::ContourModel::New();

  mitk::Point3D p;
  p[0] = p[1] = p[2] = 0;

  contour->AddVertex(p);

  //same point is used here so the epsilon can be chosen very small
  contour->SelectVertexAt(p, 0.01);

  MITK_TEST_CONDITION(contour->GetSelectedVertex() != NULL, "Vertex was selected at index");
}

//move a vertex by a translation vector
static void TestMoveSelectedVertex()
{
  mitk::ContourModel::Pointer contour = mitk::ContourModel::New();

  mitk::Point3D p;
  p[0] = p[1] = p[2] = 0;

  contour->AddVertex(p);

  //same point is used here so the epsilon can be chosen very small
  contour->SelectVertexAt(p, 0.01);

  mitk::Vector3D v;
  v[0] = 1;
  v[1] = 3;
  v[2] = -1;

  contour->MoveSelectedVertex(v);

  const mitk::ContourModel::VertexType* vertex = contour->GetSelectedVertex();

  bool correctlyMoved = false;

  correctlyMoved = (vertex->Coordinates)[0] == (v[0]) &&
                   (vertex->Coordinates)[1] == (v[1]) &&
                   (vertex->Coordinates)[2] == (v[2]);

  MITK_TEST_CONDITION(correctlyMoved, "Vertex has been moved");
}

//test to move the whole contour
static void TestMoveContour()
{
  mitk::ContourModel::Pointer contour = mitk::ContourModel::New();

  mitk::Point3D p;
  p[0] = p[1] = p[2] = 0;

  contour->AddVertex(p);

  mitk::Point3D p2;
  p2[0] = p2[1] = p2[2] = 0;

  contour->AddVertex(p2);

  mitk::Vector3D v;
  v[0] = 1;
  v[1] = 3;
  v[2] = -1;

  contour->MoveContour(v);

  mitk::ContourModel::VertexIterator it = contour->IteratorBegin();
  mitk::ContourModel::VertexIterator end = contour->IteratorEnd();

  bool correctlyMoved = false;

  while(it != end)
  {

    correctlyMoved &= (*it)->Coordinates[0] == (v[0]) &&
      (*it)->Coordinates[1] == (v[1]) &&
      (*it)->Coordinates[2] == (v[2]);
  }

  MITK_TEST_CONDITION(correctlyMoved, "Contour has been moved");
}

//remove a vertex by index
static void TestRemoveVertexAtIndex()
{
  mitk::ContourModel::Pointer contour = mitk::ContourModel::New();

  mitk::Point3D p;
  p[0] = p[1] = p[2] = 0;

  contour->AddVertex(p);

  contour->RemoveVertexAt(0);

  MITK_TEST_CONDITION(contour->GetNumberOfVertices() == 0, "removed vertex");
}

//remove a vertex by position
static void TestRemoveVertexAtWorldPosition()
{
  mitk::ContourModel::Pointer contour = mitk::ContourModel::New();

  mitk::Point3D p;
  p[0] = p[1] = p[2] = 0;

  contour->AddVertex(p);

  contour->RemoveVertexAt(p, 0.01);

  MITK_TEST_CONDITION(contour->GetNumberOfVertices() == 0, "removed vertex");
}

//check closeable contour
static void TestIsclosed()
{
    mitk::ContourModel::Pointer contour = mitk::ContourModel::New();

  mitk::Point3D p;
  p[0] = p[1] = p[2] = 0;

  contour->AddVertex(p);

  mitk::Point3D p2;
  p2[0] = p2[1] = p2[2] = 1;

  contour->AddVertex(p2);

  contour->Close();

  MITK_TEST_CONDITION(contour->IsClosed(), "close contour");
}

//test concatenating two contours
static void TestConcatenate()
{
  mitk::ContourModel::Pointer contour = mitk::ContourModel::New();

  mitk::Point3D p;
  p[0] = p[1] = p[2] = 0;

  contour->AddVertex(p);

  mitk::Point3D p2;
  p2[0] = p2[1] = p2[2] = 1;

  contour->AddVertex(p2);

  mitk::ContourModel::Pointer contour2 = mitk::ContourModel::New();

  mitk::Point3D p3;
  p3[0] = -2;
  p3[1] = 10;
  p3[2] = 0;

  contour2->AddVertex(p3);

  mitk::Point3D p4;
  p4[0] = -3;
  p4[1] = 6;
  p4[2] = -5;

  contour2->AddVertex(p4);

  contour->Concatenate(contour2);

  MITK_TEST_CONDITION(contour->GetNumberOfVertices() == 4, "two contours were concatenated");
}

int mitkContourModelTest(int argc, char* argv[])
{
  MITK_TEST_BEGIN("mitkContourModelTest")

    TestAddVertex();
  TestSelectVertexAtIndex();
  TestSelectVertexAtWorldposition();
  TestMoveSelectedVertex();
  TestRemoveVertexAtIndex();
  TestRemoveVertexAtWorldPosition();
  TestIsclosed();
  TestConcatenate();

  MITK_TEST_END()
}<|endoftext|>


<commit_before>/*=========================================================================

Program:   Visualization Toolkit

Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
All rights reserved.
See Copyright.txt or http://www.kitware.com/Copyright.htm for details.

This software is distributed WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE.  See the above copyright notice for more information.

=========================================================================*/
#include "vtkLCMGLProp.h"

#include <vtkObjectFactory.h>
#include <vtkSmartPointer.h>
#include <vtkActor.h>

#include <vtkOpenGL.h>

#include <lcmtypes/bot_lcmgl/data_t.hpp>
#include <bot_lcmgl_render/lcmgl_decode.h>

//----------------------------------------------------------------------------
vtkStandardNewMacro(vtkLCMGLProp);

class vtkLCMGLProp::vtkInternal {
public:
  vtkInternal()
    {
      this->GLData.datalen = 0;
    }

  bot_lcmgl::data_t GLData;

  std::vector<vtkSmartPointer<vtkActor> > Actors;
};

//----------------------------------------------------------------------------
vtkLCMGLProp::vtkLCMGLProp()
{
  this->Internal = new vtkInternal;
}

//----------------------------------------------------------------------------
vtkLCMGLProp::~vtkLCMGLProp()
{
  delete this->Internal;
}

//----------------------------------------------------------------------------
double* vtkLCMGLProp::GetBounds()
{
  static double bounds[6] = {-5, 5, -5, 5, -5, 5};
  return bounds;
}

//----------------------------------------------------------------------------
void vtkLCMGLProp::UpdateGLData(const char* messageData)
{
  int status = this->Internal->GLData.decode(messageData, 0, 1e6);
  if (!status)
    {
    this->Internal->GLData.name = std::string();
    this->Internal->GLData.datalen = 0;
    }
}

//----------------------------------------------------------------------------
void vtkLCMGLProp::ReleaseGraphicsResources(vtkWindow *w)
{
  for (size_t i = 0; i < this->Internal->Actors.size(); ++i)
    {
    this->Internal->Actors[i]->ReleaseGraphicsResources(w);
    }
}

//----------------------------------------------------------------------------
int vtkLCMGLProp::RenderOpaqueGeometry(vtkViewport *v)
{
  if (this->Internal->GLData.datalen)
    {
    glPushMatrix();
    glPushAttrib(GL_ENABLE_BIT);
    glDisable(GL_LIGHTING);

    bot_lcmgl_decode(&this->Internal->GLData.data.front(), this->Internal->GLData.datalen);

    glPopAttrib ();
    glPopMatrix();
    return 1;
    }

  int count=0;

  for (size_t i = 0; i < this->Internal->Actors.size(); ++i)
    {
    count += this->Internal->Actors[i]->RenderOpaqueGeometry(v);
    }

  return count;
}

//----------------------------------------------------------------------------
int vtkLCMGLProp::RenderOverlay(vtkViewport *v)
{
  int count=0;

  for (size_t i = 0; i < this->Internal->Actors.size(); ++i)
    {
    count += this->Internal->Actors[i]->RenderOverlay(v);
    }

  return count;
}

//----------------------------------------------------------------------------
int vtkLCMGLProp::RenderTranslucentPolygonalGeometry(vtkViewport *v)
{
  int count=0;

  for (size_t i = 0; i < this->Internal->Actors.size(); ++i)
    {
    count += this->Internal->Actors[i]->RenderTranslucentPolygonalGeometry(v);
    }

  return count;
}

//----------------------------------------------------------------------------
int vtkLCMGLProp::HasTranslucentPolygonalGeometry()
{
  int result = 0;

  for (size_t i = 0; i < this->Internal->Actors.size(); ++i)
    {
    result |= this->Internal->Actors[i]->HasTranslucentPolygonalGeometry();
    }

  return result;
}

//----------------------------------------------------------------------------
void vtkLCMGLProp::PrintSelf(ostream& os, vtkIndent indent)
{
  this->Superclass::PrintSelf(os,indent);
}
<commit_msg>add more setup to the GL state machine before calling into lcmgl<commit_after>/*=========================================================================

Program:   Visualization Toolkit

Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
All rights reserved.
See Copyright.txt or http://www.kitware.com/Copyright.htm for details.

This software is distributed WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE.  See the above copyright notice for more information.

=========================================================================*/
#include "vtkLCMGLProp.h"

#include <vtkObjectFactory.h>
#include <vtkSmartPointer.h>
#include <vtkActor.h>

#include <vtkOpenGL.h>

#include <lcmtypes/bot_lcmgl/data_t.hpp>
#include <bot_lcmgl_render/lcmgl_decode.h>

//----------------------------------------------------------------------------
vtkStandardNewMacro(vtkLCMGLProp);

class vtkLCMGLProp::vtkInternal {
public:
  vtkInternal()
    {
      this->GLData.datalen = 0;
    }

  bot_lcmgl::data_t GLData;

  std::vector<vtkSmartPointer<vtkActor> > Actors;
};

//----------------------------------------------------------------------------
vtkLCMGLProp::vtkLCMGLProp()
{
  this->Internal = new vtkInternal;
}

//----------------------------------------------------------------------------
vtkLCMGLProp::~vtkLCMGLProp()
{
  delete this->Internal;
}

//----------------------------------------------------------------------------
double* vtkLCMGLProp::GetBounds()
{
  static double bounds[6] = {-5, 5, -5, 5, -5, 5};
  return bounds;
}

//----------------------------------------------------------------------------
void vtkLCMGLProp::UpdateGLData(const char* messageData)
{
  int status = this->Internal->GLData.decode(messageData, 0, 1e6);
  if (!status)
    {
    this->Internal->GLData.name = std::string();
    this->Internal->GLData.datalen = 0;
    }
}

//----------------------------------------------------------------------------
void vtkLCMGLProp::ReleaseGraphicsResources(vtkWindow *w)
{
  for (size_t i = 0; i < this->Internal->Actors.size(); ++i)
    {
    this->Internal->Actors[i]->ReleaseGraphicsResources(w);
    }
}

//----------------------------------------------------------------------------
int vtkLCMGLProp::RenderOpaqueGeometry(vtkViewport *v)
{
  if (this->Internal->GLData.datalen)
    {
    glPushMatrix();
    glPushAttrib(GL_ENABLE_BIT | GL_POINT_BIT | GL_POLYGON_STIPPLE_BIT |
                 GL_POLYGON_BIT | GL_LINE_BIT | GL_FOG_BIT | GL_LIGHTING_BIT);

    // The goal here is to setup the GL state machine to match what the
    // default libbot viewer uses.  At this point, the VTK graphichs
    // engine may be in some unknown state due to whatever has already been
    // drawn.  In order to set the GL state machine to match libbot, I have
    // reviewed code in:
    //   my_draw() of bot_lcmgl_render/lcmgl_decode.c
    //   render_scene() of bot_vis/viewer.c

    // reset lighting and color flags
    glDisable(GL_LIGHTING);
    glDisable(GL_COLOR_MATERIAL);
    glDisable(GL_LIGHT0);
    glDisable(GL_LIGHT1);
    glDisable(GL_LIGHT2);
    glDisable(GL_LIGHT3);
    glDisable(GL_LIGHT4);
    glDisable(GL_LIGHT5);
    glDisable(GL_LIGHT6);

    // reset lighting model
    float light_model_ambient[] = {0.2, 0.2, 0.2, 1.0};
    glLightModelfv(GL_LIGHT_MODEL_AMBIENT, light_model_ambient);

    // reset GL_LIGHT0
    float light0_amb[] = { 0.0, 0.0, 0.0, 1 };
    float light0_dif[] = { 1, 1, 1, 1 };
    float light0_spe[] = { .5, .5, .5, 1 };
    float light0_pos[] = { 100, 100, 100, 0 };
    float spot_exp[] = {0.0};
    float spot_cutoff[] = {180.0};
    float att0[] = {0.0};
    float att1[] = {1.0};
    glLightfv (GL_LIGHT0, GL_AMBIENT, light0_amb);
    glLightfv (GL_LIGHT0, GL_DIFFUSE, light0_dif);
    glLightfv (GL_LIGHT0, GL_SPECULAR, light0_spe);
    glLightfv (GL_LIGHT0, GL_SPOT_EXPONENT, spot_exp);
    glLightfv (GL_LIGHT0, GL_SPOT_CUTOFF, spot_cutoff);
    glLightfv (GL_LIGHT0, GL_CONSTANT_ATTENUATION, att1);
    glLightfv (GL_LIGHT0, GL_LINEAR_ATTENUATION, att1);
    glLightfv (GL_LIGHT0, GL_QUADRATIC_ATTENUATION, att1);

    // don't set the light position, use the position that VTK has already set
    //glLightfv (GL_LIGHT0, GL_POSITION, light0_pos);

    // enable GL_LIGHT0
    glEnable (GL_LIGHT0);

    // this makes the rendering prettier for lines and points
    if (1)
    {
      glEnable(GL_LINE_STIPPLE);
      glEnable(GL_LINE_SMOOTH);
      glHint(GL_LINE_SMOOTH_HINT, GL_NICEST);
      glEnable(GL_POINT_SMOOTH);
      glHint(GL_POINT_SMOOTH_HINT, GL_NICEST);
    }

    // enable transparency, copied from my_draw() of bot_lcmgl_render/lcmgl_decode.c
    glEnable(GL_BLEND);
    glEnable(GL_DEPTH_TEST);

    // invoke lcmgl rendering
    bot_lcmgl_decode(&this->Internal->GLData.data.front(), this->Internal->GLData.datalen);

    glPopAttrib ();
    glPopMatrix();
    return 1;
    }

  int count=0;

  for (size_t i = 0; i < this->Internal->Actors.size(); ++i)
    {
    count += this->Internal->Actors[i]->RenderOpaqueGeometry(v);
    }

  return count;
}

//----------------------------------------------------------------------------
int vtkLCMGLProp::RenderOverlay(vtkViewport *v)
{
  int count=0;

  for (size_t i = 0; i < this->Internal->Actors.size(); ++i)
    {
    count += this->Internal->Actors[i]->RenderOverlay(v);
    }

  return count;
}

//----------------------------------------------------------------------------
int vtkLCMGLProp::RenderTranslucentPolygonalGeometry(vtkViewport *v)
{
  int count=0;

  for (size_t i = 0; i < this->Internal->Actors.size(); ++i)
    {
    count += this->Internal->Actors[i]->RenderTranslucentPolygonalGeometry(v);
    }

  return count;
}

//----------------------------------------------------------------------------
int vtkLCMGLProp::HasTranslucentPolygonalGeometry()
{
  int result = 0;

  for (size_t i = 0; i < this->Internal->Actors.size(); ++i)
    {
    result |= this->Internal->Actors[i]->HasTranslucentPolygonalGeometry();
    }

  return result;
}

//----------------------------------------------------------------------------
void vtkLCMGLProp::PrintSelf(ostream& os, vtkIndent indent)
{
  this->Superclass::PrintSelf(os,indent);
}
<|endoftext|>


<commit_before>#include "product_price_trend.h"

#include <common/Utilities.h>
#include <log-manager/PriceHistory.h>
#include <document-manager/DocumentManager.h>

#include <am/range/AmIterator.h>
#include <libcassandra/util_functions.h>
#include <glog/logging.h>

#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/filesystem.hpp>
#include <algorithm>

using namespace std;
using namespace libcassandra;
using izenelib::util::UString;

namespace sf1r
{

ProductPriceTrend::ProductPriceTrend(
        const boost::shared_ptr<DocumentManager>& document_manager,
        const CassandraStorageConfig& cassandraConfig,
        const string& data_dir,
        const vector<string>& group_prop_vec,
        const vector<uint32_t>& time_int_vec)
    : document_manager_(document_manager)
    , cassandraConfig_(cassandraConfig)
    , data_dir_(data_dir)
    , group_prop_vec_(group_prop_vec)
    , time_int_vec_(time_int_vec)
    , enable_tpc_(!group_prop_vec_.empty() && !time_int_vec_.empty())
{
}

ProductPriceTrend::~ProductPriceTrend()
{
    for (TPCStorage::iterator it = tpc_storage_.begin();
            it != tpc_storage_.end(); ++it)
    {
        for (uint32_t i = 0; i < time_int_vec_.size(); i++)
            delete it->second[i];
    }
}

bool ProductPriceTrend::Init()
{
    if (!cassandraConfig_.enable)
        return false;

    price_history_.reset(new PriceHistory(cassandraConfig_.keyspace));
    price_history_->createColumnFamily();

    if (!price_history_->is_enabled)
        return false;

    for (vector<string>::const_iterator it = group_prop_vec_.begin();
            it != group_prop_vec_.end(); ++it)
    {
        vector<TPCBTree *>& prop_tpc = tpc_storage_[*it];
        for (uint32_t i = 0; i < time_int_vec_.size(); i++)
        {
            string db_path = data_dir_ + "/" + *it + "." + boost::lexical_cast<string>(time_int_vec_[i]) + ".tpc";
            prop_tpc.push_back(new TPCBTree(db_path));
            if (!prop_tpc.back()->open())
            {
                boost::filesystem::remove_all(db_path);
                prop_tpc.back()->open();
            }
        }
    }
    price_history_buffer_.reserve(2000);

    return true;
}

bool ProductPriceTrend::Insert(
        const string& docid,
        const ProductPrice& price,
        time_t timestamp)
{
    price_history_buffer_.push_back(PriceHistoryRow(docid));
    price_history_buffer_.back().insert(timestamp, price);

    if (IsBufferFull_())
    {
        return Flush();
    }

    return true;
}

bool ProductPriceTrend::Update(
        const string& docid,
        const ProductPrice& price,
        time_t timestamp,
        map<string, string>& group_prop_map)
{
    price_history_buffer_.push_back(PriceHistoryRow(docid));
    price_history_buffer_.back().insert(timestamp, price);

    if (enable_tpc_ && !group_prop_map.empty() && price.value.first > 0)
    {
        PropItemType& prop_item = prop_map_[docid];
        prop_item.first = price.value.first;
        prop_item.second.swap(group_prop_map);
    }

    if (IsBufferFull_())
    {
        return Flush();
    }

    return true;
}

bool ProductPriceTrend::IsBufferFull_()
{
    return price_history_buffer_.size() >= 2000;
}

bool ProductPriceTrend::Flush()
{
    bool ret = true;
    time_t now = Utilities::createTimeStamp();

    if (!prop_map_.empty())
    {
        for (uint32_t i = 0; i < time_int_vec_.size(); i++)
        {
            if (!UpdateTPC_(i, now))
                ret = false;
        }

        prop_map_.clear();
    }

    if (!price_history_->updateMultiRow(price_history_buffer_))
        ret = false;

    price_history_buffer_.clear();

    return ret;
}

bool ProductPriceTrend::CronJob()
{
    if (!enable_tpc_) return true;

    //TODO
    return true;
}

bool ProductPriceTrend::UpdateTPC_(uint32_t time_int, time_t timestamp)
{
    vector<string> key_list;
    Utilities::getKeyList(key_list, prop_map_);
    if (timestamp == -1)
        timestamp = Utilities::createTimeStamp();
    timestamp -= 86400000000LL * time_int_vec_[time_int];

    vector<PriceHistoryRow> row_list;
    if (!price_history_->getMultiSlice(row_list, key_list, serializeLong(timestamp), "", 1))
        return false;

    map<string, map<string, TPCQueue> > tpc_cache;
    for (uint32_t i = 0; i < row_list.size(); i++)
    {
        const PropItemType& prop_item = prop_map_.find(row_list[i].getDocId())->second;
        const pair<time_t, ProductPrice>& price_record = *(row_list[i].getPriceHistory().begin());
        const ProductPriceType& old_price = price_record.second.value.first;
        float price_cut = old_price == 0 ? 0 : 1 - prop_item.first / old_price;

        for (map<string, string>::const_iterator it = prop_item.second.begin();
                it != prop_item.second.end(); ++it)
        {
            TPCQueue& tpc_queue = tpc_cache[it->first][it->second];
            if (tpc_queue.empty())
            {
                tpc_queue.reserve(1000);
                tpc_storage_[it->first][time_int]->get(it->second, tpc_queue);
            }

            if (tpc_queue.size() < 1000)
            {
                tpc_queue.push_back(make_pair(price_cut, row_list[i].getDocId()));
                push_heap(tpc_queue.begin(), tpc_queue.end(), rel_ops::operator> <TPCQueue::value_type>);
            }
            else if (price_cut > tpc_queue[0].first)
            {
                pop_heap(tpc_queue.begin(), tpc_queue.end(), rel_ops::operator> <TPCQueue::value_type>);
                tpc_queue.back().first = price_cut;
                tpc_queue.back().second = row_list[i].getDocId();
                push_heap(tpc_queue.begin(), tpc_queue.end(), rel_ops::operator> <TPCQueue::value_type>);
            }
        }
    }

    for (map<string, map<string, TPCQueue> >::const_iterator it = tpc_cache.begin();
            it != tpc_cache.end(); ++it)
    {
        TPCBTree* tpc_btree = tpc_storage_[it->first][time_int];
        for (map<string, TPCQueue>::const_iterator mit = it->second.begin();
                mit != it->second.end(); ++mit)
        {
            tpc_btree->update(mit->first, mit->second);
        }
        tpc_btree->flush();
    }

    return true;
}

bool ProductPriceTrend::GetMultiPriceHistory(
        PriceHistoryList& history_list,
        const vector<string>& docid_list,
        time_t from_tt,
        time_t to_tt,
        string& error_msg)
{
    string from_str(from_tt == -1 ? "" : serializeLong(from_tt));
    string to_str(to_tt == -1 ? "" : serializeLong(to_tt));

    vector<PriceHistoryRow> row_list;
    if (!price_history_->getMultiSlice(row_list, docid_list, from_str, to_str))
    {
        error_msg = "Failed retrieving price histories from Cassandra";
        return false;
    }

    history_list.reserve(history_list.size() + row_list.size());
    for (vector<PriceHistoryRow>::const_iterator it = row_list.begin();
            it != row_list.end(); ++it)
    {
        const PriceHistory::PriceHistoryType& price_history = it->getPriceHistory();
        PriceHistoryItem history_item;
        history_item.reserve(price_history.size());
        for (PriceHistory::PriceHistoryType::const_iterator hit = price_history.begin();
                hit != price_history.end(); ++hit)
        {
            history_item.push_back(make_pair(string(), hit->second.value));
            history_item.back().first = boost::posix_time::to_iso_string(
                    boost::posix_time::from_time_t(hit->first / 1000000 - timezone)
                    + boost::posix_time::microseconds(hit->first % 1000000));
        }
        history_list.push_back(make_pair(it->getDocId(), PriceHistoryItem()));
        history_list.back().second.swap(history_item);
    }

    if (history_list.empty())
    {
        error_msg = "Have not got any valid docid";
        return false;
    }

    return true;
}

bool ProductPriceTrend::GetMultiPriceRange(
        PriceRangeList& range_list,
        const vector<string>& docid_list,
        time_t from_tt,
        time_t to_tt,
        string& error_msg)
{
    string from_str(from_tt == -1 ? "" : serializeLong(from_tt));
    string to_str(to_tt == -1 ? "" : serializeLong(to_tt));

    vector<PriceHistoryRow> row_list;
    if (!price_history_->getMultiSlice(row_list, docid_list, from_str, to_str))
    {
        error_msg = "Failed retrieving price histories from Cassandra";
        return false;
    }

    range_list.reserve(range_list.size() + row_list.size());
    for (vector<PriceHistoryRow>::const_iterator it = row_list.begin();
            it != row_list.end(); ++it)
    {
        const PriceHistory::PriceHistoryType& price_history = it->getPriceHistory();
        ProductPrice range_item;
        for (PriceHistory::PriceHistoryType::const_iterator hit = price_history.begin();
                hit != price_history.end(); ++hit)
        {
            range_item += hit->second;
        }
        range_list.push_back(make_pair(it->getDocId(), range_item.value));
    }

    if (range_list.empty())
    {
        error_msg = "Have not got any valid docid";
        return false;
    }

    return true;
}

bool ProductPriceTrend::GetTopPriceCutList(
        TPCQueue& tpc_queue,
        const string& prop_name,
        const string& prop_value,
        uint32_t days,
        uint32_t count,
        string& error_msg)
{
    if (!enable_tpc_)
    {
        error_msg = "Top price-cut list is not enabled for this collection.";
        return false;
    }

    TPCStorage::const_iterator tpc_it = tpc_storage_.find(prop_name);
    if (tpc_it == tpc_storage_.end())
    {
        error_msg = "Don't find data for this property name: " + prop_name;
        return false;
    }

    uint32_t time_int = 0;
    while (days > time_int_vec_[time_int++] && time_int < time_int_vec_.size() - 1);
    if (!tpc_it->second[time_int]->get(prop_value, tpc_queue))
    {
        error_msg = "Don't find price-cut record for this property value: " + prop_value;
        return false;
    }

    sort_heap(tpc_queue.begin(), tpc_queue.end(), rel_ops::operator><TPCQueue::value_type>);
    if (tpc_queue.size() > count)
        tpc_queue.resize(count);

    return true;
}

bool ProductPriceTrend::MigratePriceHistory(
        const string& new_keyspace,
        const string& old_prefix,
        const string& new_prefix,
        uint32_t start,
        string& error_msg)
{
    if (new_keyspace == cassandraConfig_.keyspace)
        return true;

    vector<string> docid_list;
    docid_list.reserve(1000);
    boost::shared_ptr<PriceHistory> new_price_history(new PriceHistory(new_keyspace));
    new_price_history->createColumnFamily();

    uint32_t count = 0;
    for (uint32_t i = start; i <= document_manager_->getMaxDocId(); i++)
    {
        Document doc;
        document_manager_->getDocument(i, doc);
        Document::property_const_iterator kit = doc.findProperty("DOCID");
        if (kit == doc.propertyEnd())
            continue;

        const UString& key = kit->second.get<UString>();
        docid_list.push_back(string());
        key.convertString(docid_list.back(), UString::UTF_8);
        if (!old_prefix.empty())
        {
            docid_list.back() = old_prefix + "_" + docid_list.back();
        }

        if (docid_list.size() == 1000)
        {
            vector<PriceHistoryRow> row_list;
            if (!price_history_->getMultiSlice(row_list, docid_list))
            {
                error_msg = "Can't get price history from Cassandra.";
                return false;
            }

            if (!old_prefix.empty())
            {
                std::size_t len = old_prefix.length() + 1;
                if (!new_prefix.empty())
                {
                    for (vector<PriceHistoryRow>::iterator it = row_list.begin();
                            it != row_list.end(); ++it)
                    {
                        it->setDocId(new_prefix + "_" + it->getDocId().substr(len));
                    }
                }
                else
                {
                    for (vector<PriceHistoryRow>::iterator it = row_list.begin();
                            it != row_list.end(); ++it)
                    {
                        it->setDocId(it->getDocId().substr(len));
                    }
                }
            }
            else if (!new_prefix.empty())
            {
                for (vector<PriceHistoryRow>::iterator it = row_list.begin();
                        it != row_list.end(); ++it)
                {
                    it->setDocId(new_prefix + "_" + it->getDocId());
                }
            }

            if (!new_price_history->updateMultiRow(row_list))
            {
                error_msg = "Can't update price history to Cassandra.";
                return false;
            }
            docid_list.clear();
        }

        if (++count % 100000 == 0)
        {
            LOG(INFO) << "Migrating price history: " << count;
        }
    }

    if (!docid_list.empty())
    {
        vector<PriceHistoryRow> row_list;
        if (!price_history_->getMultiSlice(row_list, docid_list))
        {
            error_msg = "Can't get price history from Cassandra.";
            return false;
        }

        if (!old_prefix.empty())
        {
            std::size_t len = old_prefix.length() + 1;
            if (!new_prefix.empty())
            {
                for (vector<PriceHistoryRow>::iterator it = row_list.begin();
                        it != row_list.end(); ++it)
                {
                    it->setDocId(new_prefix + "_" + it->getDocId().substr(len));
                }
            }
            else
            {
                for (vector<PriceHistoryRow>::iterator it = row_list.begin();
                        it != row_list.end(); ++it)
                {
                    it->setDocId(it->getDocId().substr(len));
                }
            }
        }
        else if (!new_prefix.empty())
        {
            for (vector<PriceHistoryRow>::iterator it = row_list.begin();
                    it != row_list.end(); ++it)
            {
                it->setDocId(new_prefix + "_" + it->getDocId());
            }
        }
        if (!new_price_history->updateMultiRow(row_list))
        {
            error_msg = "Can't update price history to Cassandra.";
            return false;
        }
    }
    LOG(INFO) << "Successfully migrated price history for " << count << " products.";

    return true;
}

}
<commit_msg>tiny fix<commit_after>#include "product_price_trend.h"

#include <common/Utilities.h>
#include <log-manager/PriceHistory.h>
#include <document-manager/DocumentManager.h>

#include <am/range/AmIterator.h>
#include <libcassandra/util_functions.h>
#include <glog/logging.h>

#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/filesystem.hpp>
#include <algorithm>

using namespace std;
using namespace libcassandra;
using izenelib::util::UString;

namespace sf1r
{

ProductPriceTrend::ProductPriceTrend(
        const boost::shared_ptr<DocumentManager>& document_manager,
        const CassandraStorageConfig& cassandraConfig,
        const string& data_dir,
        const vector<string>& group_prop_vec,
        const vector<uint32_t>& time_int_vec)
    : document_manager_(document_manager)
    , cassandraConfig_(cassandraConfig)
    , data_dir_(data_dir)
    , group_prop_vec_(group_prop_vec)
    , time_int_vec_(time_int_vec)
    , enable_tpc_(!group_prop_vec_.empty() && !time_int_vec_.empty())
{
}

ProductPriceTrend::~ProductPriceTrend()
{
    for (TPCStorage::iterator it = tpc_storage_.begin();
            it != tpc_storage_.end(); ++it)
    {
        for (uint32_t i = 0; i < time_int_vec_.size(); i++)
            delete it->second[i];
    }
}

bool ProductPriceTrend::Init()
{
    if (!cassandraConfig_.enable)
        return false;

    price_history_.reset(new PriceHistory(cassandraConfig_.keyspace));
    price_history_->createColumnFamily();

    if (!price_history_->is_enabled)
        return false;

    for (vector<string>::const_iterator it = group_prop_vec_.begin();
            it != group_prop_vec_.end(); ++it)
    {
        vector<TPCBTree *>& prop_tpc = tpc_storage_[*it];
        for (uint32_t i = 0; i < time_int_vec_.size(); i++)
        {
            string db_path = data_dir_ + "/" + *it + "." + boost::lexical_cast<string>(time_int_vec_[i]) + ".tpc";
            prop_tpc.push_back(new TPCBTree(db_path));
            if (!prop_tpc.back()->open())
            {
                boost::filesystem::remove_all(db_path);
                prop_tpc.back()->open();
            }
        }
    }
    price_history_buffer_.reserve(2000);

    return true;
}

bool ProductPriceTrend::Insert(
        const string& docid,
        const ProductPrice& price,
        time_t timestamp)
{
    price_history_buffer_.push_back(PriceHistoryRow(docid));
    price_history_buffer_.back().insert(timestamp, price);

    if (IsBufferFull_())
    {
        return Flush();
    }

    return true;
}

bool ProductPriceTrend::Update(
        const string& docid,
        const ProductPrice& price,
        time_t timestamp,
        map<string, string>& group_prop_map)
{
    price_history_buffer_.push_back(PriceHistoryRow(docid));
    price_history_buffer_.back().insert(timestamp, price);

    if (enable_tpc_ && !group_prop_map.empty() && price.value.first > 0)
    {
        PropItemType& prop_item = prop_map_[docid];
        prop_item.first = price.value.first;
        prop_item.second.swap(group_prop_map);
    }

    if (IsBufferFull_())
    {
        return Flush();
    }

    return true;
}

bool ProductPriceTrend::IsBufferFull_()
{
    return price_history_buffer_.size() >= 2000;
}

bool ProductPriceTrend::Flush()
{
    bool ret = true;
    time_t now = Utilities::createTimeStamp();

    if (!prop_map_.empty())
    {
        for (uint32_t i = 0; i < time_int_vec_.size(); i++)
        {
            if (!UpdateTPC_(i, now))
                ret = false;
        }

        PropMapType().swap(prop_map_);
    }

    if (!price_history_->updateMultiRow(price_history_buffer_))
        ret = false;

    price_history_buffer_.clear();

    return ret;
}

bool ProductPriceTrend::CronJob()
{
    if (!enable_tpc_) return true;

    //TODO
    return true;
}

bool ProductPriceTrend::UpdateTPC_(uint32_t time_int, time_t timestamp)
{
    vector<string> key_list;
    Utilities::getKeyList(key_list, prop_map_);
    if (timestamp == -1)
        timestamp = Utilities::createTimeStamp();
    timestamp -= 86400000000LL * time_int_vec_[time_int];

    vector<PriceHistoryRow> row_list;
    if (!price_history_->getMultiSlice(row_list, key_list, serializeLong(timestamp), "", 1))
        return false;

    map<string, map<string, TPCQueue> > tpc_cache;
    for (uint32_t i = 0; i < row_list.size(); i++)
    {
        const PropItemType& prop_item = prop_map_.find(row_list[i].getDocId())->second;
        const pair<time_t, ProductPrice>& price_record = *(row_list[i].getPriceHistory().begin());
        const ProductPriceType& old_price = price_record.second.value.first;
        float price_cut = old_price == 0 ? 0 : 1 - prop_item.first / old_price;

        for (map<string, string>::const_iterator it = prop_item.second.begin();
                it != prop_item.second.end(); ++it)
        {
            TPCQueue& tpc_queue = tpc_cache[it->first][it->second];
            if (tpc_queue.empty())
            {
                tpc_queue.reserve(1000);
                tpc_storage_[it->first][time_int]->get(it->second, tpc_queue);
            }

            if (tpc_queue.size() < 1000)
            {
                tpc_queue.push_back(make_pair(price_cut, row_list[i].getDocId()));
                push_heap(tpc_queue.begin(), tpc_queue.end(), rel_ops::operator> <TPCQueue::value_type>);
            }
            else if (price_cut > tpc_queue[0].first)
            {
                pop_heap(tpc_queue.begin(), tpc_queue.end(), rel_ops::operator> <TPCQueue::value_type>);
                tpc_queue.back().first = price_cut;
                tpc_queue.back().second = row_list[i].getDocId();
                push_heap(tpc_queue.begin(), tpc_queue.end(), rel_ops::operator> <TPCQueue::value_type>);
            }
        }
    }

    for (map<string, map<string, TPCQueue> >::const_iterator it = tpc_cache.begin();
            it != tpc_cache.end(); ++it)
    {
        TPCBTree* tpc_btree = tpc_storage_[it->first][time_int];
        for (map<string, TPCQueue>::const_iterator mit = it->second.begin();
                mit != it->second.end(); ++mit)
        {
            tpc_btree->update(mit->first, mit->second);
        }
        tpc_btree->flush();
    }

    return true;
}

bool ProductPriceTrend::GetMultiPriceHistory(
        PriceHistoryList& history_list,
        const vector<string>& docid_list,
        time_t from_tt,
        time_t to_tt,
        string& error_msg)
{
    string from_str(from_tt == -1 ? "" : serializeLong(from_tt));
    string to_str(to_tt == -1 ? "" : serializeLong(to_tt));

    vector<PriceHistoryRow> row_list;
    if (!price_history_->getMultiSlice(row_list, docid_list, from_str, to_str))
    {
        error_msg = "Failed retrieving price histories from Cassandra";
        return false;
    }

    history_list.reserve(history_list.size() + row_list.size());
    for (vector<PriceHistoryRow>::const_iterator it = row_list.begin();
            it != row_list.end(); ++it)
    {
        const PriceHistory::PriceHistoryType& price_history = it->getPriceHistory();
        PriceHistoryItem history_item;
        history_item.reserve(price_history.size());
        for (PriceHistory::PriceHistoryType::const_iterator hit = price_history.begin();
                hit != price_history.end(); ++hit)
        {
            history_item.push_back(make_pair(string(), hit->second.value));
            history_item.back().first = boost::posix_time::to_iso_string(
                    boost::posix_time::from_time_t(hit->first / 1000000 - timezone)
                    + boost::posix_time::microseconds(hit->first % 1000000));
        }
        history_list.push_back(make_pair(it->getDocId(), PriceHistoryItem()));
        history_list.back().second.swap(history_item);
    }

    if (history_list.empty())
    {
        error_msg = "Have not got any valid docid";
        return false;
    }

    return true;
}

bool ProductPriceTrend::GetMultiPriceRange(
        PriceRangeList& range_list,
        const vector<string>& docid_list,
        time_t from_tt,
        time_t to_tt,
        string& error_msg)
{
    string from_str(from_tt == -1 ? "" : serializeLong(from_tt));
    string to_str(to_tt == -1 ? "" : serializeLong(to_tt));

    vector<PriceHistoryRow> row_list;
    if (!price_history_->getMultiSlice(row_list, docid_list, from_str, to_str))
    {
        error_msg = "Failed retrieving price histories from Cassandra";
        return false;
    }

    range_list.reserve(range_list.size() + row_list.size());
    for (vector<PriceHistoryRow>::const_iterator it = row_list.begin();
            it != row_list.end(); ++it)
    {
        const PriceHistory::PriceHistoryType& price_history = it->getPriceHistory();
        ProductPrice range_item;
        for (PriceHistory::PriceHistoryType::const_iterator hit = price_history.begin();
                hit != price_history.end(); ++hit)
        {
            range_item += hit->second;
        }
        range_list.push_back(make_pair(it->getDocId(), range_item.value));
    }

    if (range_list.empty())
    {
        error_msg = "Have not got any valid docid";
        return false;
    }

    return true;
}

bool ProductPriceTrend::GetTopPriceCutList(
        TPCQueue& tpc_queue,
        const string& prop_name,
        const string& prop_value,
        uint32_t days,
        uint32_t count,
        string& error_msg)
{
    if (!enable_tpc_)
    {
        error_msg = "Top price-cut list is not enabled for this collection.";
        return false;
    }

    TPCStorage::const_iterator tpc_it = tpc_storage_.find(prop_name);
    if (tpc_it == tpc_storage_.end())
    {
        error_msg = "Don't find data for this property name: " + prop_name;
        return false;
    }

    uint32_t time_int = 0;
    while (days > time_int_vec_[time_int++] && time_int < time_int_vec_.size() - 1);
    if (!tpc_it->second[time_int]->get(prop_value, tpc_queue))
    {
        error_msg = "Don't find price-cut record for this property value: " + prop_value;
        return false;
    }

    sort_heap(tpc_queue.begin(), tpc_queue.end(), rel_ops::operator><TPCQueue::value_type>);
    if (tpc_queue.size() > count)
        tpc_queue.resize(count);

    return true;
}

bool ProductPriceTrend::MigratePriceHistory(
        const string& new_keyspace,
        const string& old_prefix,
        const string& new_prefix,
        uint32_t start,
        string& error_msg)
{
    if (new_keyspace == cassandraConfig_.keyspace)
        return true;

    vector<string> docid_list;
    docid_list.reserve(1000);
    boost::shared_ptr<PriceHistory> new_price_history(new PriceHistory(new_keyspace));
    new_price_history->createColumnFamily();

    uint32_t count = 0;
    for (uint32_t i = start; i <= document_manager_->getMaxDocId(); i++)
    {
        Document doc;
        document_manager_->getDocument(i, doc);
        Document::property_const_iterator kit = doc.findProperty("DOCID");
        if (kit == doc.propertyEnd())
            continue;

        const UString& key = kit->second.get<UString>();
        docid_list.push_back(string());
        key.convertString(docid_list.back(), UString::UTF_8);
        if (!old_prefix.empty())
        {
            docid_list.back() = old_prefix + "_" + docid_list.back();
        }

        if (docid_list.size() == 1000)
        {
            vector<PriceHistoryRow> row_list;
            if (!price_history_->getMultiSlice(row_list, docid_list))
            {
                error_msg = "Can't get price history from Cassandra.";
                return false;
            }

            if (!old_prefix.empty())
            {
                std::size_t len = old_prefix.length() + 1;
                if (!new_prefix.empty())
                {
                    for (vector<PriceHistoryRow>::iterator it = row_list.begin();
                            it != row_list.end(); ++it)
                    {
                        it->setDocId(new_prefix + "_" + it->getDocId().substr(len));
                    }
                }
                else
                {
                    for (vector<PriceHistoryRow>::iterator it = row_list.begin();
                            it != row_list.end(); ++it)
                    {
                        it->setDocId(it->getDocId().substr(len));
                    }
                }
            }
            else if (!new_prefix.empty())
            {
                for (vector<PriceHistoryRow>::iterator it = row_list.begin();
                        it != row_list.end(); ++it)
                {
                    it->setDocId(new_prefix + "_" + it->getDocId());
                }
            }

            if (!new_price_history->updateMultiRow(row_list))
            {
                error_msg = "Can't update price history to Cassandra.";
                return false;
            }
            docid_list.clear();
        }

        if (++count % 100000 == 0)
        {
            LOG(INFO) << "Migrating price history: " << count;
        }
    }

    if (!docid_list.empty())
    {
        vector<PriceHistoryRow> row_list;
        if (!price_history_->getMultiSlice(row_list, docid_list))
        {
            error_msg = "Can't get price history from Cassandra.";
            return false;
        }

        if (!old_prefix.empty())
        {
            std::size_t len = old_prefix.length() + 1;
            if (!new_prefix.empty())
            {
                for (vector<PriceHistoryRow>::iterator it = row_list.begin();
                        it != row_list.end(); ++it)
                {
                    it->setDocId(new_prefix + "_" + it->getDocId().substr(len));
                }
            }
            else
            {
                for (vector<PriceHistoryRow>::iterator it = row_list.begin();
                        it != row_list.end(); ++it)
                {
                    it->setDocId(it->getDocId().substr(len));
                }
            }
        }
        else if (!new_prefix.empty())
        {
            for (vector<PriceHistoryRow>::iterator it = row_list.begin();
                    it != row_list.end(); ++it)
            {
                it->setDocId(new_prefix + "_" + it->getDocId());
            }
        }
        if (!new_price_history->updateMultiRow(row_list))
        {
            error_msg = "Can't update price history to Cassandra.";
            return false;
        }
    }
    LOG(INFO) << "Successfully migrated price history for " << count << " products.";

    return true;
}

}
<|endoftext|>


<commit_before>#include "tests-base.hpp"

#include "utf8rewind.h"

extern "C" {
	#include "../internal/database.h"
};

#include "helpers-strings.hpp"

TEST(Utf8IsNormalizedDecompose, BasicLatinSingle)
{
	/*
		U+0053
		     Y
		     0
	*/

	const char* i = "S";
	size_t is = strlen(i);
	size_t o = 0;

	EXPECT_EQ(UTF8_NORMALIZATION_RESULT_YES, utf8isnormalized(i, is, UTF8_NORMALIZE_DECOMPOSE, &o));
	EXPECT_EQ(1, o);
}

TEST(Utf8IsNormalizedDecompose, BasicLatinMultiple)
{
	/*
		U+0050 U+006F U+006E U+0079
		     Y      Y      Y      Y
		     0      0      0      0
	*/

	const char* i = "Pony";
	size_t is = strlen(i);
	size_t o = 0;

	EXPECT_EQ(UTF8_NORMALIZATION_RESULT_YES, utf8isnormalized(i, is, UTF8_NORMALIZE_DECOMPOSE, &o));
	EXPECT_EQ(4, o);
}

TEST(Utf8IsNormalizedDecompose, BasicLatinCompatibility)
{
	/*
		U+0061 U+0072 U+0063 U+0068
		     Y      Y      Y      Y
		     0      0      0      0
	*/

	const char* i = "arch";
	size_t is = strlen(i);
	size_t o = 0;

	EXPECT_EQ(UTF8_NORMALIZATION_RESULT_YES, utf8isnormalized(i, is, UTF8_NORMALIZE_DECOMPOSE | UTF8_NORMALIZE_COMPATIBILITY, &o));
	EXPECT_EQ(4, o);
}<commit_msg>suite-utf8-isnormalized-decompose: Added tests for multi-byte YES results.<commit_after>#include "tests-base.hpp"

#include "utf8rewind.h"

extern "C" {
	#include "../internal/database.h"
};

#include "helpers-strings.hpp"

TEST(Utf8IsNormalizedDecompose, BasicLatinSingle)
{
	/*
		U+0053
		     Y
		     0
	*/

	const char* i = "S";
	size_t is = strlen(i);
	size_t o = 0;

	EXPECT_EQ(UTF8_NORMALIZATION_RESULT_YES, utf8isnormalized(i, is, UTF8_NORMALIZE_DECOMPOSE, &o));
	EXPECT_EQ(1, o);
}

TEST(Utf8IsNormalizedDecompose, BasicLatinMultiple)
{
	/*
		U+0050 U+006F U+006E U+0079
		     Y      Y      Y      Y
		     0      0      0      0
	*/

	const char* i = "Pony";
	size_t is = strlen(i);
	size_t o = 0;

	EXPECT_EQ(UTF8_NORMALIZATION_RESULT_YES, utf8isnormalized(i, is, UTF8_NORMALIZE_DECOMPOSE, &o));
	EXPECT_EQ(4, o);
}

TEST(Utf8IsNormalizedDecompose, BasicLatinCompatibility)
{
	/*
		U+0061 U+0072 U+0063 U+0068
		     Y      Y      Y      Y
		     0      0      0      0
	*/

	const char* i = "arch";
	size_t is = strlen(i);
	size_t o = 0;

	EXPECT_EQ(UTF8_NORMALIZATION_RESULT_YES, utf8isnormalized(i, is, UTF8_NORMALIZE_DECOMPOSE | UTF8_NORMALIZE_COMPATIBILITY, &o));
	EXPECT_EQ(4, o);
}

TEST(Utf8IsNormalizedDecompose, MultiByteYesSingle)
{
	/*
		U+2205
		     Y
		     0
	*/

	const char* i = "\xE2\x88\x85";
	size_t is = strlen(i);
	size_t o = 0;

	EXPECT_EQ(UTF8_NORMALIZATION_RESULT_YES, utf8isnormalized(i, is, UTF8_NORMALIZE_DECOMPOSE, &o));
	EXPECT_EQ(3, o);
}

TEST(Utf8IsNormalizedDecompose, MultiByteYesMultiple)
{
	/*
		U+21A9 U+225D U+21F0 U+0338
		     Y      Y      Y      Y
		     0      0      0      0
	*/

	const char* i = "\xE2\x86\xA9\xE2\x89\x9D\xE2\x87\xB0\xCC\xB8";
	size_t is = strlen(i);
	size_t o = 0;

	EXPECT_EQ(UTF8_NORMALIZATION_RESULT_YES, utf8isnormalized(i, is, UTF8_NORMALIZE_DECOMPOSE, &o));
	EXPECT_EQ(11, o);
}

TEST(Utf8IsNormalizedDecompose, MultiByteYesUnordered)
{
	/*
		U+031A U+0F35 U+0328
		     Y      Y      Y
		   232    220    202
	*/

	const char* i = "\xCC\x9A\xE0\xBC\xB5\xCC\xA8";
	size_t is = strlen(i);
	size_t o = 0;

	EXPECT_EQ(UTF8_NORMALIZATION_RESULT_NO, utf8isnormalized(i, is, UTF8_NORMALIZE_DECOMPOSE, &o));
	EXPECT_EQ(7, o);
}

TEST(Utf8IsNormalizedDecompose, MultiByteYesCompatibility)
{
	/*
		U+027B U+05C0 U+07EB
		     Y      Y      Y
		     0      0    230
	*/

	const char* i = "\xC9\xBB\xD7\x80\xDF\xAB";
	size_t is = strlen(i);
	size_t o = 0;

	EXPECT_EQ(UTF8_NORMALIZATION_RESULT_YES, utf8isnormalized(i, is, UTF8_NORMALIZE_DECOMPOSE | UTF8_NORMALIZE_COMPATIBILITY, &o));
	EXPECT_EQ(6, o);
}<|endoftext|>


<commit_before>/*
 * Source position file
 *
 * This file is part of the "XieXie-Compiler" (Copyright (c) 2013 by Lukas Hermanns)
 * See "LICENSE.txt" for license information.
 */

#include "SourcePosition.h"
#include "StringModifier.h"

namespace SyntaxAnalyzer
{

const SourcePosition SourcePosition::ignore;

SourcePosition::SourcePosition(unsigned int row, unsigned int column) :
    row_   ( row    ),
    column_( column )
{
}

std::string SourcePosition::ToString() const
{
    return ToStr(row_) + ":" + ToStr(column_);
}

void SourcePosition::IncRow()
{
    ++row_;
    column_ = 0;
}
void SourcePosition::IncColumn()
{
    ++column_;
}

bool SourcePosition::IsValid() const
{
    return row_ > 0 && column_ > 0;
}

void SourcePosition::Reset()
{
    row_ = column_ = 0;
}

} // /namespace SyntaxAnalyzer

// ================================================================================<commit_msg>- Added adjustment to 'SourcePosition::ToString' column value for LL(k = 1) parser.<commit_after>/*
 * Source position file
 *
 * This file is part of the "XieXie-Compiler" (Copyright (c) 2013 by Lukas Hermanns)
 * See "LICENSE.txt" for license information.
 */

#include "SourcePosition.h"
#include "StringModifier.h"

namespace SyntaxAnalyzer
{

const SourcePosition SourcePosition::ignore;

SourcePosition::SourcePosition(unsigned int row, unsigned int column) :
    row_   ( row    ),
    column_( column )
{
}

std::string SourcePosition::ToString() const
{
    return ToStr(row_) + ":" + ToStr(column_ > 0 ? column_ - 1 : 0);
}

void SourcePosition::IncRow()
{
    ++row_;
    column_ = 0;
}

void SourcePosition::IncColumn()
{
    ++column_;
}

bool SourcePosition::IsValid() const
{
    return row_ > 0 && column_ > 0;
}

void SourcePosition::Reset()
{
    row_ = column_ = 0;
}

} // /namespace SyntaxAnalyzer

// ================================================================================<|endoftext|>


<commit_before>#include "MQTTWrapper.h"

MQTTWrapper::MQTTWrapper(std::string id, std::string host, int port) :
    mosqpp::mosquittopp(id.c_str(), false), host(host), port(port) {}

void MQTTWrapper::begin(void) {
    mosqpp::lib_init();
    printf("Connecting to MQTT\n");
    this->connect(this->host.c_str(), this->port, 60 /* keepalive */);
    this->subscribe(nullptr, "/sensornet/in/#");
}

void MQTTWrapper::end(void) {
    mosqpp::lib_cleanup();
}

void MQTTWrapper::loop(void) {
  mosqpp::mosquittopp::loop();
}

void MQTTWrapper::send_message(std::string subject, std::string body) {
    this->publish(nullptr, subject.c_str(), body.length(), body.c_str(), 0);
}

void MQTTWrapper::set_on_message_callback(on_msg_cb cb) {
    this->cb = cb;
}

void MQTTWrapper::on_disconnect(int mid) {
    printf("Disconnecting from MQTT\n");
    this->end();
    sleep(15);
    this->begin();
}

void MQTTWrapper::on_message(const struct mosquitto_message *message) {
    auto subject = std::string(message->topic);
    auto body = std::string(reinterpret_cast<char *>(message->payload));
    this->cb(subject, body);
}
<commit_msg>Use empty string if MQTT payload is empty<commit_after>#include "MQTTWrapper.h"

MQTTWrapper::MQTTWrapper(std::string id, std::string host, int port) :
    mosqpp::mosquittopp(id.c_str(), false), host(host), port(port) {}

void MQTTWrapper::begin(void) {
    mosqpp::lib_init();
    printf("Connecting to MQTT\n");
    this->connect(this->host.c_str(), this->port, 60 /* keepalive */);
    this->subscribe(nullptr, "/sensornet/in/#");
}

void MQTTWrapper::end(void) {
    mosqpp::lib_cleanup();
}

void MQTTWrapper::loop(void) {
  mosqpp::mosquittopp::loop();
}

void MQTTWrapper::send_message(std::string subject, std::string body) {
    this->publish(nullptr, subject.c_str(), body.length(), body.c_str(), 0);
}

void MQTTWrapper::set_on_message_callback(on_msg_cb cb) {
    this->cb = cb;
}

void MQTTWrapper::on_disconnect(int mid) {
    printf("Disconnecting from MQTT\n");
    this->end();
    sleep(15);
    this->begin();
}

void MQTTWrapper::on_message(const struct mosquitto_message *message) {
    auto subject = std::string(message->topic);
    auto body = message->payloadlen > 0 ?
        std::string(reinterpret_cast<char *>(message->payload)) : std::string("");
    this->cb(subject, body);
}
<|endoftext|>


<commit_before>/*
 * Copyright 2019 The Project Oak Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "oak/server/module_invocation.h"

#include "absl/memory/memory.h"
#include "asylo/util/logging.h"
#include "oak/common/policy.h"
#include "oak/proto/grpc_encap.pb.h"
#include "oak/proto/policy.pb.h"
#include "oak/server/channel.h"

namespace oak {

namespace {

// Copy the data from a gRPC ByteBuffer into a Message.
std::unique_ptr<Message> Unwrap(const grpc::ByteBuffer& buffer) {
  std::vector<::grpc::Slice> slices;
  grpc::Status status = buffer.Dump(&slices);
  if (!status.ok()) {
    LOG(QFATAL) << "Could not unwrap buffer";
  }
  auto msg = absl::make_unique<Message>();
  for (const auto& slice : slices) {
    msg->data.insert(msg->data.end(), slice.begin(), slice.end());
  }
  return msg;
}

}  // namespace

void ModuleInvocation::Start() {
  auto callback = new std::function<void(bool)>(
      std::bind(&ModuleInvocation::ReadRequest, this, std::placeholders::_1));
  LOG(INFO) << "invocation#" << stream_id_ << " Start: request service->RequestCall => ReadRequest";
  service_->RequestCall(&context_, &stream_, queue_, queue_, callback);
}

void ModuleInvocation::ReadRequest(bool ok) {
  if (!ok) {
    LOG(INFO) << "invocation#" << stream_id_ << " ReadRequest: not OK, terminating";
    delete this;
    return;
  }
  LOG(INFO) << "invocation#" << stream_id_
            << " ReadRequest: request stream->Read => ProcessRequest";
  auto callback = new std::function<void(bool)>(
      std::bind(&ModuleInvocation::ProcessRequest, this, std::placeholders::_1));
  stream_.Read(&request_, callback);

  // Now that processing of this request has started, start watching out for the
  // next request.
  LOG(INFO) << "invocation#" << stream_id_ << " start next invocation";
  auto next_invocation = new ModuleInvocation(service_, queue_, grpc_node_);
  next_invocation->Start();
}

void ModuleInvocation::ProcessRequest(bool ok) {
  if (!ok) {
    LOG(INFO) << "invocation#" << stream_id_ << " ProcessRequest: not OK, terminating";
    delete this;
    return;
  }
  std::unique_ptr<Message> request_msg = Unwrap(request_);

  LOG(INFO) << "invocation#" << stream_id_
            << " ProcessRequest: handling gRPC call: " << context_.method();
  for (auto entry : context_.client_metadata()) {
    auto key = entry.first;
    auto value = entry.second;
    // Redact authorization bearer tokens.
    if (key == kOakAuthorizationBearerTokenGrpcMetadataKey) {
      value = "<redacted>";
    }
    LOG(INFO) << "invocation#" << stream_id_ << " gRPC client metadata: [" << key << "] -> ["
              << value << "]";
  }

  // Build an encapsulation of the gRPC request invocation and put it in a Message.
  oak::GrpcRequest grpc_request;
  grpc_request.set_method_name(context_.method());
  google::protobuf::Any* any = new google::protobuf::Any();
  any->set_value(request_msg->data.data(), request_msg->data.size());
  grpc_request.set_allocated_req_msg(any);
  grpc_request.set_last(true);

  std::unique_ptr<Message> req_msg = absl::make_unique<Message>();
  size_t serialized_size = grpc_request.ByteSizeLong();
  req_msg->data.resize(serialized_size);
  grpc_request.SerializeToArray(req_msg->data.data(), req_msg->data.size());
  {
    auto range = context_.client_metadata().equal_range(kOakPolicyGrpcMetadataKey);
    for (auto entry = range.first; entry != range.second; ++entry) {
      std::string policy_base64(entry->second.data(), entry->second.size());
      oak::policy::Label policy = DeserializePolicy(policy_base64);
      // TODO(https://github.com/project-oak/oak/issues/306): Note that at the moment policies may
      // refer to authentication bearer tokens, which if logged in this way may be reused by
      // unauthorized parties. For now we are fine with this, eventually bearer tokens will be
      // removed and replaced by public key assertions, in which case it will always be safe to log
      // policies.
      LOG(INFO) << "invocation#" << stream_id_ << " Oak policy: " << policy.DebugString();
      req_msg->label = policy;
    }
  }
  {
    // TODO: Provide capability to declassify data for any authorization token provided.
    auto range =
        context_.client_metadata().equal_range(kOakAuthorizationBearerTokenGrpcMetadataKey);
    for (auto entry = range.first; entry != range.second; ++entry) {
      // Redact authorization bearer tokens.
      LOG(INFO) << "invocation#" << stream_id_ << " Oak Authorization Token: <redacted>";
    }
  }

  // Create a pair of channels for communication corresponding to this
  // particular method invocation, one for sending in requests, and one for
  // receiving responses.
  MessageChannel::ChannelHalves req_halves = MessageChannel::Create();
  req_half_ = std::move(req_halves.write);
  MessageChannel::ChannelHalves rsp_halves = MessageChannel::Create();
  rsp_half_ = std::move(rsp_halves.read);

  // Build a notification message that just holds references to these two
  // newly-created channels.
  std::unique_ptr<Message> notify_msg = absl::make_unique<Message>();
  notify_msg->channels.push_back(absl::make_unique<ChannelHalf>(std::move(req_halves.read)));
  notify_msg->channels.push_back(absl::make_unique<ChannelHalf>(std::move(rsp_halves.write)));

  // Write the request message to the just-created request channel.
  req_half_->Write(std::move(req_msg));
  LOG(INFO) << "invocation#" << stream_id_
            << " ProcessRequest: Wrote encapsulated request to new gRPC request channel";

  // Write the notification message to the gRPC input channel, which the runtime
  // connected to the Node.
  MessageChannelWriteHalf* notify_half = grpc_node_->BorrowWriteChannel();
  notify_half->Write(std::move(notify_msg));
  LOG(INFO) << "invocation#" << stream_id_
            << " ProcessRequest: Wrote notification request to gRPC input channel";

  // Move straight onto sending first response.
  SendResponse(true);
}

void ModuleInvocation::SendResponse(bool ok) {
  if (!ok) {
    LOG(INFO) << "invocation#" << stream_id_ << " SendResponse: not OK, terminating";
    delete this;
    return;
  }

  // Do the work of sending a response in a separate thread, because we may
  // need to block (to wait for a streaming server response) and don't want
  // to hold up the main completion queue thread.
  std::thread other([this] { BlockingSendResponse(); });
  other.detach();
}

void ModuleInvocation::BlockingSendResponse() {
  ReadResult rsp_result;
  // Block until we can read a single queued GrpcResponse message (in serialized form) from the
  // gRPC output channel.
  LOG(INFO) << "invocation#" << stream_id_ << " SendResponse: do blocking-read on grpc channel";
  rsp_result = rsp_half_->BlockingRead(INT_MAX, INT_MAX);
  if (rsp_result.required_size > 0) {
    LOG(ERROR) << "invocation#" << stream_id_
               << " SendResponse: Message size too large: " << rsp_result.required_size;
    FinishAndCleanUp(grpc::Status(grpc::StatusCode::INTERNAL, "Message size too large"));
    return;
  }

  LOG(INFO) << "invocation#" << stream_id_ << " SendResponse: Read encapsulated message of size "
            << rsp_result.msg->data.size() << " from gRPC output channel";
  oak::GrpcResponse grpc_response;
  if (!grpc_response.ParseFromString(
          std::string(rsp_result.msg->data.data(), rsp_result.msg->data.size()))) {
    LOG(ERROR) << "invocation#" << stream_id_
               << " SendResponse: failed to parse encapsulated message";
    FinishAndCleanUp(grpc::Status(grpc::StatusCode::INTERNAL, "Message failed to parse"));
    return;
  }
  // Any channel references included with the message will be dropped.

  const grpc::string& inner_msg = grpc_response.rsp_msg().value();
  grpc::Slice slice(inner_msg.data(), inner_msg.size());
  grpc::ByteBuffer bb(&slice, /*nslices=*/1);

  grpc::WriteOptions options;
  if (!grpc_response.last()) {
    // More response data is expected, so queue up another SendResponse action.
    LOG(INFO) << "invocation#" << stream_id_ << " SendResponse: Non-final inner response of size "
              << inner_msg.size() << ", request stream->Write => SendResponse";
    auto callback = new std::function<void(bool)>(
        std::bind(&ModuleInvocation::SendResponse, this, std::placeholders::_1));
    stream_.Write(bb, options, callback);
  } else if (!grpc_response.has_rsp_msg()) {
    // Final iteration but no response, just Finish.
    LOG(INFO) << "invocation#" << stream_id_ << " SendResponse: Final inner response empty";
    FinishAndCleanUp(::grpc::Status::OK);
  } else {
    // Final response, so WriteAndFinish.
    LOG(INFO) << "invocation#" << stream_id_ << " SendResponse: Final inner response of size "
              << inner_msg.size() << ", request stream->WriteAndFinish => CleanUp";
    options.set_last_message();
    auto callback = new std::function<void(bool)>(
        std::bind(&ModuleInvocation::CleanUp, this, std::placeholders::_1));
    stream_.WriteAndFinish(bb, options, ::grpc::Status::OK, callback);
  }
}

void ModuleInvocation::CleanUp(bool /*ok*/) {
  LOG(INFO) << "invocation#" << stream_id_ << " CleanUp: delete self";
  delete this;
}

void ModuleInvocation::FinishAndCleanUp(const grpc::Status& status) {
  // Finish the current invocation (which triggers self-destruction).
  LOG(INFO) << "invocation#" << stream_id_
            << " FinishAndCleanUp: request stream->Finish => CleanUp";
  auto callback = new std::function<void(bool)>(
      std::bind(&ModuleInvocation::CleanUp, this, std::placeholders::_1));
  stream_.Finish(status, callback);
}

}  // namespace oak
<commit_msg>runtime: pass gRPC response error status through<commit_after>/*
 * Copyright 2019 The Project Oak Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "oak/server/module_invocation.h"

#include "absl/memory/memory.h"
#include "asylo/util/logging.h"
#include "oak/common/policy.h"
#include "oak/proto/grpc_encap.pb.h"
#include "oak/proto/policy.pb.h"
#include "oak/server/channel.h"

namespace oak {

namespace {

// Copy the data from a gRPC ByteBuffer into a Message.
std::unique_ptr<Message> Unwrap(const grpc::ByteBuffer& buffer) {
  std::vector<::grpc::Slice> slices;
  grpc::Status status = buffer.Dump(&slices);
  if (!status.ok()) {
    LOG(QFATAL) << "Could not unwrap buffer";
  }
  auto msg = absl::make_unique<Message>();
  for (const auto& slice : slices) {
    msg->data.insert(msg->data.end(), slice.begin(), slice.end());
  }
  return msg;
}

}  // namespace

void ModuleInvocation::Start() {
  auto callback = new std::function<void(bool)>(
      std::bind(&ModuleInvocation::ReadRequest, this, std::placeholders::_1));
  LOG(INFO) << "invocation#" << stream_id_ << " Start: request service->RequestCall => ReadRequest";
  service_->RequestCall(&context_, &stream_, queue_, queue_, callback);
}

void ModuleInvocation::ReadRequest(bool ok) {
  if (!ok) {
    LOG(INFO) << "invocation#" << stream_id_ << " ReadRequest: not OK, terminating";
    delete this;
    return;
  }
  LOG(INFO) << "invocation#" << stream_id_
            << " ReadRequest: request stream->Read => ProcessRequest";
  auto callback = new std::function<void(bool)>(
      std::bind(&ModuleInvocation::ProcessRequest, this, std::placeholders::_1));
  stream_.Read(&request_, callback);

  // Now that processing of this request has started, start watching out for the
  // next request.
  LOG(INFO) << "invocation#" << stream_id_ << " start next invocation";
  auto next_invocation = new ModuleInvocation(service_, queue_, grpc_node_);
  next_invocation->Start();
}

void ModuleInvocation::ProcessRequest(bool ok) {
  if (!ok) {
    LOG(INFO) << "invocation#" << stream_id_ << " ProcessRequest: not OK, terminating";
    delete this;
    return;
  }
  std::unique_ptr<Message> request_msg = Unwrap(request_);

  LOG(INFO) << "invocation#" << stream_id_
            << " ProcessRequest: handling gRPC call: " << context_.method();
  for (auto entry : context_.client_metadata()) {
    auto key = entry.first;
    auto value = entry.second;
    // Redact authorization bearer tokens.
    if (key == kOakAuthorizationBearerTokenGrpcMetadataKey) {
      value = "<redacted>";
    }
    LOG(INFO) << "invocation#" << stream_id_ << " gRPC client metadata: [" << key << "] -> ["
              << value << "]";
  }

  // Build an encapsulation of the gRPC request invocation and put it in a Message.
  oak::GrpcRequest grpc_request;
  grpc_request.set_method_name(context_.method());
  google::protobuf::Any* any = new google::protobuf::Any();
  any->set_value(request_msg->data.data(), request_msg->data.size());
  grpc_request.set_allocated_req_msg(any);
  grpc_request.set_last(true);

  std::unique_ptr<Message> req_msg = absl::make_unique<Message>();
  size_t serialized_size = grpc_request.ByteSizeLong();
  req_msg->data.resize(serialized_size);
  grpc_request.SerializeToArray(req_msg->data.data(), req_msg->data.size());
  {
    auto range = context_.client_metadata().equal_range(kOakPolicyGrpcMetadataKey);
    for (auto entry = range.first; entry != range.second; ++entry) {
      std::string policy_base64(entry->second.data(), entry->second.size());
      oak::policy::Label policy = DeserializePolicy(policy_base64);
      // TODO(https://github.com/project-oak/oak/issues/306): Note that at the moment policies may
      // refer to authentication bearer tokens, which if logged in this way may be reused by
      // unauthorized parties. For now we are fine with this, eventually bearer tokens will be
      // removed and replaced by public key assertions, in which case it will always be safe to log
      // policies.
      LOG(INFO) << "invocation#" << stream_id_ << " Oak policy: " << policy.DebugString();
      req_msg->label = policy;
    }
  }
  {
    // TODO: Provide capability to declassify data for any authorization token provided.
    auto range =
        context_.client_metadata().equal_range(kOakAuthorizationBearerTokenGrpcMetadataKey);
    for (auto entry = range.first; entry != range.second; ++entry) {
      // Redact authorization bearer tokens.
      LOG(INFO) << "invocation#" << stream_id_ << " Oak Authorization Token: <redacted>";
    }
  }

  // Create a pair of channels for communication corresponding to this
  // particular method invocation, one for sending in requests, and one for
  // receiving responses.
  MessageChannel::ChannelHalves req_halves = MessageChannel::Create();
  req_half_ = std::move(req_halves.write);
  MessageChannel::ChannelHalves rsp_halves = MessageChannel::Create();
  rsp_half_ = std::move(rsp_halves.read);

  // Build a notification message that just holds references to these two
  // newly-created channels.
  std::unique_ptr<Message> notify_msg = absl::make_unique<Message>();
  notify_msg->channels.push_back(absl::make_unique<ChannelHalf>(std::move(req_halves.read)));
  notify_msg->channels.push_back(absl::make_unique<ChannelHalf>(std::move(rsp_halves.write)));

  // Write the request message to the just-created request channel.
  req_half_->Write(std::move(req_msg));
  LOG(INFO) << "invocation#" << stream_id_
            << " ProcessRequest: Wrote encapsulated request to new gRPC request channel";

  // Write the notification message to the gRPC input channel, which the runtime
  // connected to the Node.
  MessageChannelWriteHalf* notify_half = grpc_node_->BorrowWriteChannel();
  notify_half->Write(std::move(notify_msg));
  LOG(INFO) << "invocation#" << stream_id_
            << " ProcessRequest: Wrote notification request to gRPC input channel";

  // Move straight onto sending first response.
  SendResponse(true);
}

void ModuleInvocation::SendResponse(bool ok) {
  if (!ok) {
    LOG(INFO) << "invocation#" << stream_id_ << " SendResponse: not OK, terminating";
    delete this;
    return;
  }

  // Do the work of sending a response in a separate thread, because we may
  // need to block (to wait for a streaming server response) and don't want
  // to hold up the main completion queue thread.
  std::thread other([this] { BlockingSendResponse(); });
  other.detach();
}

void ModuleInvocation::BlockingSendResponse() {
  ReadResult rsp_result;
  // Block until we can read a single queued GrpcResponse message (in serialized form) from the
  // gRPC output channel.
  LOG(INFO) << "invocation#" << stream_id_ << " SendResponse: do blocking-read on grpc channel";
  rsp_result = rsp_half_->BlockingRead(INT_MAX, INT_MAX);
  if (rsp_result.required_size > 0) {
    LOG(ERROR) << "invocation#" << stream_id_
               << " SendResponse: Message size too large: " << rsp_result.required_size;
    FinishAndCleanUp(grpc::Status(grpc::StatusCode::INTERNAL, "Message size too large"));
    return;
  }

  LOG(INFO) << "invocation#" << stream_id_ << " SendResponse: Read encapsulated message of size "
            << rsp_result.msg->data.size() << " from gRPC output channel";
  oak::GrpcResponse grpc_response;
  if (!grpc_response.ParseFromString(
          std::string(rsp_result.msg->data.data(), rsp_result.msg->data.size()))) {
    LOG(ERROR) << "invocation#" << stream_id_
               << " SendResponse: failed to parse encapsulated message";
    FinishAndCleanUp(grpc::Status(grpc::StatusCode::INTERNAL, "Message failed to parse"));
    return;
  }
  // Any channel references included with the message will be dropped.

  const grpc::string& inner_msg = grpc_response.rsp_msg().value();
  grpc::Slice slice(inner_msg.data(), inner_msg.size());
  grpc::ByteBuffer bb(&slice, /*nslices=*/1);

  grpc::WriteOptions options;
  if (!grpc_response.last()) {
    // More response data is expected, so queue up another SendResponse action.
    LOG(INFO) << "invocation#" << stream_id_ << " SendResponse: Non-final inner response of size "
              << inner_msg.size() << ", request stream->Write => SendResponse";
    auto callback = new std::function<void(bool)>(
        std::bind(&ModuleInvocation::SendResponse, this, std::placeholders::_1));
    stream_.Write(bb, options, callback);
  } else if (!grpc_response.has_rsp_msg()) {
    // Final iteration but no response, just Finish.
    google::rpc::Status status = grpc_response.status();
    LOG(INFO) << "invocation#" << stream_id_ << " SendResponse: Final inner response empty, status "
              << status.code();
    FinishAndCleanUp(grpc::Status(static_cast<grpc::StatusCode>(status.code()), status.message()));
  } else {
    // Final response, so WriteAndFinish.
    LOG(INFO) << "invocation#" << stream_id_ << " SendResponse: Final inner response of size "
              << inner_msg.size() << ", request stream->WriteAndFinish => CleanUp";
    options.set_last_message();
    auto callback = new std::function<void(bool)>(
        std::bind(&ModuleInvocation::CleanUp, this, std::placeholders::_1));
    stream_.WriteAndFinish(bb, options, ::grpc::Status::OK, callback);
  }
}

void ModuleInvocation::CleanUp(bool /*ok*/) {
  LOG(INFO) << "invocation#" << stream_id_ << " CleanUp: delete self";
  delete this;
}

void ModuleInvocation::FinishAndCleanUp(const grpc::Status& status) {
  // Finish the current invocation (which triggers self-destruction).
  LOG(INFO) << "invocation#" << stream_id_
            << " FinishAndCleanUp: request stream->Finish => CleanUp";
  auto callback = new std::function<void(bool)>(
      std::bind(&ModuleInvocation::CleanUp, this, std::placeholders::_1));
  stream_.Finish(status, callback);
}

}  // namespace oak
<|endoftext|>