<commit_before>/*****************************************************************************
 * Media Library
 *****************************************************************************
 * Copyright (C) 2015-2019 Hugo Beauzée-Luyssen, Videolabs, VideoLAN
 *
 * Authors: Hugo Beauzée-Luyssen <hugo@beauzee.fr>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

#if HAVE_CONFIG_H
# include "config.h"
#endif

#include "ParserWorker.h"
#include "Parser.h"
#include "Media.h"
#include "Folder.h"

namespace medialibrary
{
namespace parser
{

Worker::Worker()
    : m_parserCb( nullptr )
    , m_stopParser( false )
    , m_paused( false )
    , m_idle( true )
{
}

void Worker::start()
{
    // This function is called from a locked context.

    // Ensure we don't start multiple times.
    assert( m_thread.joinable() == false );
    m_thread = compat::Thread{ &Worker::mainloop, this };
}

void Worker::pause()
{
    std::lock_guard<compat::Mutex> lock( m_lock );
    m_paused = true;
}

void Worker::resume()
{
    std::lock_guard<compat::Mutex> lock( m_lock );
    m_paused = false;
    m_cond.notify_all();
}

void Worker::signalStop()
{
    if ( m_thread.joinable() )
    {
        std::lock_guard<compat::Mutex> lock( m_lock );
        m_cond.notify_all();
        m_stopParser = true;
    }
    m_service->stop();
}

void Worker::stop()
{
    if ( m_thread.joinable() == true )
        m_thread.join();
}

void Worker::parse( std::shared_ptr<Task> t )
{
    // Avoid flickering from idle/not idle when not many tasks are running.
    // The thread calling parse for the next parser step might not have
    // something left to do and would turn idle, potentially causing all
    // services to be idle for a very short time, until this parser
    // thread awakes/starts, causing the global parser idle state to be
    // restored back to false.
    // Since we are queuing a task, we already know that this thread is
    // not idle
    setIdle( false );

    // Even if no threads appear to be started, we need to lock this in case
    // we're currently doing a stop/start
    {
        std::lock_guard<compat::Mutex> lock( m_lock );
        if ( m_thread.get_id() == compat::Thread::id{} )
        {
            m_tasks.push( std::move( t ) );
            start();
            return;
        }
        m_tasks.push( std::move( t ) );
    }
    m_cond.notify_all();
}

bool Worker::initialize( MediaLibrary* ml, IParserCb* parserCb,
                               std::shared_ptr<IParserService> service )
{
    m_ml = ml;
    m_service = std::move( service );
    m_parserCb = parserCb;
    // Run the service specific initializer
    return m_service->initialize( ml );
}

bool Worker::isIdle() const
{
    return m_idle;
}

void Worker::flush()
{
    std::unique_lock<compat::Mutex> lock( m_lock );
    assert( m_paused == true || m_thread.get_id() == compat::Thread::id{} );
    m_idleCond.wait( lock, [this]() {
        return m_idle == true;
    });
    while ( m_tasks.empty() == false )
        m_tasks.pop();
    m_service->onFlushing();
}

void Worker::restart()
{
    m_service->onRestarted();
}

void Worker::mainloop()
{
    // It would be unsafe to call name() at the end of this function, since
    // we might stop the thread during ParserService destruction. This implies
    // that the underlying service has been deleted already.
    std::string serviceName = m_service->name();
    LOG_INFO("Entering ParserService [", serviceName, "] thread");
    setIdle( false );

    while ( true )
    {
        std::shared_ptr<Task> task;
        {
            std::unique_lock<compat::Mutex> lock( m_lock );
            if ( m_stopParser == true )
                break;
            if ( m_tasks.empty() == true || m_paused == true )
            {
                LOG_DEBUG( "Halting ParserService [", serviceName, "] mainloop" );
                setIdle( true );
                m_idleCond.notify_all();
                m_cond.wait( lock, [this]() {
                    return ( m_tasks.empty() == false && m_paused == false )
                            || m_stopParser == true;
                });
                LOG_DEBUG( "Resuming ParserService [", serviceName, "] mainloop" );
                // We might have been woken up because the parser is being destroyed
                if ( m_stopParser  == true )
                    break;
                setIdle( false );
            }
            // Otherwise it's safe to assume we have at least one element.
            LOG_DEBUG('[', serviceName, "] has ", m_tasks.size(), " tasks remaining" );
            task = std::move( m_tasks.front() );
            m_tasks.pop();
        }
        // Special case to restore uncompleted tasks from a parser thread
        if ( task == nullptr )
        {
            restoreTasks();
            continue;
        }
        if ( task->isStepCompleted( m_service->targetedStep() ) == true )
        {
            LOG_DEBUG( "Skipping completed task [", serviceName, "] on ", task->mrl() );
            m_parserCb->done( std::move( task ), Status::Success );
            continue;
        }
        Status status;
        try
        {
            LOG_DEBUG( "Executing ", serviceName, " task on ", task->mrl() );
            auto chrono = std::chrono::steady_clock::now();
            auto file = std::static_pointer_cast<File>( task->file() );
            auto media = std::static_pointer_cast<Media>( task->media() );

            if ( file != nullptr && file->isRemovable() )
            {
                auto folder = Folder::fetch( m_ml, file->folderId() );
                if ( folder->isPresent() == false )
                {
                    LOG_DEBUG( "Postponing parsing of ", file->rawMrl(),
                              " until the device containing it gets mounted back" );
                    m_parserCb->done( std::move( task ), Status::TemporaryUnavailable );
                    continue;
                }
            }
            task->startParserStep();
            status = m_service->run( *task );
            auto duration = std::chrono::steady_clock::now() - chrono;
            LOG_DEBUG( "Done executing ", serviceName, " task on ", task->mrl(), " in ",
                       std::chrono::duration_cast<std::chrono::milliseconds>( duration ).count(),
                       "ms. Result: ",
                       static_cast<std::underlying_type_t<parser::Status>>( status ) );
        }
        catch ( const fs::DeviceRemovedException& )
        {
            LOG_ERROR( "Parsing of ", task->mrl(), " was interrupted "
                       "due to its containing device being unmounted" );
            status = Status::TemporaryUnavailable;
        }
        catch ( const std::exception& ex )
        {
            LOG_ERROR( "Caught an exception during ", task->mrl(), " [", serviceName, "] parsing: ", ex.what() );
            status = Status::Fatal;
        }
        if ( handleServiceResult( *task, status ) == false )
            status = Status::Fatal;
        m_parserCb->done( std::move( task ), status );
    }
    LOG_INFO("Exiting ParserService [", serviceName, "] thread");
    setIdle( true );
}

void Worker::setIdle(bool isIdle)
{
    // Calling the idleChanged callback will trigger a call to isIdle, so set the value before
    // invoking it, otherwise we have an incoherent state.
    if ( m_idle != isIdle )
    {
        m_idle = isIdle;
        m_parserCb->onIdleChanged( isIdle );
    }
}

bool Worker::handleServiceResult( Task& task, Status status )
{
    if ( status == Status::Success )
    {
        task.markStepCompleted( m_service->targetedStep() );
        // We don't want to save the extraction step in database, as restarting a
        // task with extraction completed but analysis uncompleted wouldn't run
        // the extraction again, causing the analysis to run with no info.
        if ( m_service->targetedStep() != Step::MetadataExtraction )
            return task.saveParserStep();
        // We don't want to reset the entire retry count, as we would be stuck in
        // a "loop" in case the metadata analysis fails (we'd always reset the retry
        // count to zero, then fail, then run the extraction again, reset the retry,
        // fail the analysis, and so on.
        // We can't not increment the retry count for metadata extraction, since
        // in case a file makes (lib)VLC crash, we would always try again, and
        // therefor we would keep on crashing.
        // However we don't want to just increment the retry count, since it
        // would reach the maximum value too quickly (extraction would set retry
        // count to 1, analysis to 2, and in case of failure, next run would set
        // it over 3, while we only tried 2 times. Instead we just decrement it
        // when the extraction step succeeds
        return task.decrementRetryCount();
    }
    else if ( status == Status::Completed )
    {
        task.markStepCompleted( Step::Completed );
        return task.saveParserStep();
    }
    else if ( status == Status::Discarded )
    {
        return Task::destroy( m_ml, task.id() );
    }
    return true;
}

void Worker::restoreTasks()
{
    auto tasks = Task::fetchUncompleted( m_ml );
    if ( tasks.size() > 0 )
        LOG_INFO( "Resuming parsing on ", tasks.size(), " tasks" );
    else
        LOG_DEBUG( "No task to resume." );
    for ( auto& t : tasks )
    {
        {
            std::lock_guard<compat::Mutex> lock( m_lock );
            if ( m_stopParser == true )
                break;
        }

        if ( t->restoreLinkedEntities() == false )
            continue;
        m_parserCb->parse( std::move( t ) );
    }
}

}
}
<commit_msg>ParserWorker: Minor simplification<commit_after>/*****************************************************************************
 * Media Library
 *****************************************************************************
 * Copyright (C) 2015-2019 Hugo Beauzée-Luyssen, Videolabs, VideoLAN
 *
 * Authors: Hugo Beauzée-Luyssen <hugo@beauzee.fr>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

#if HAVE_CONFIG_H
# include "config.h"
#endif

#include "ParserWorker.h"
#include "Parser.h"
#include "Media.h"
#include "Folder.h"

namespace medialibrary
{
namespace parser
{

Worker::Worker()
    : m_parserCb( nullptr )
    , m_stopParser( false )
    , m_paused( false )
    , m_idle( true )
{
}

void Worker::start()
{
    // This function is called from a locked context.

    // Ensure we don't start multiple times.
    assert( m_thread.joinable() == false );
    m_thread = compat::Thread{ &Worker::mainloop, this };
}

void Worker::pause()
{
    std::lock_guard<compat::Mutex> lock( m_lock );
    m_paused = true;
}

void Worker::resume()
{
    std::lock_guard<compat::Mutex> lock( m_lock );
    m_paused = false;
    m_cond.notify_all();
}

void Worker::signalStop()
{
    if ( m_thread.joinable() )
    {
        std::lock_guard<compat::Mutex> lock( m_lock );
        m_cond.notify_all();
        m_stopParser = true;
    }
    m_service->stop();
}

void Worker::stop()
{
    if ( m_thread.joinable() == true )
        m_thread.join();
}

void Worker::parse( std::shared_ptr<Task> t )
{
    // Avoid flickering from idle/not idle when not many tasks are running.
    // The thread calling parse for the next parser step might not have
    // something left to do and would turn idle, potentially causing all
    // services to be idle for a very short time, until this parser
    // thread awakes/starts, causing the global parser idle state to be
    // restored back to false.
    // Since we are queuing a task, we already know that this thread is
    // not idle
    setIdle( false );

    // Even if no threads appear to be started, we need to lock this in case
    // we're currently doing a stop/start
    {
        std::lock_guard<compat::Mutex> lock( m_lock );
        m_tasks.push( std::move( t ) );
        if ( m_thread.get_id() == compat::Thread::id{} )
        {
            start();
            return;
        }
    }
    m_cond.notify_all();
}

bool Worker::initialize( MediaLibrary* ml, IParserCb* parserCb,
                               std::shared_ptr<IParserService> service )
{
    m_ml = ml;
    m_service = std::move( service );
    m_parserCb = parserCb;
    // Run the service specific initializer
    return m_service->initialize( ml );
}

bool Worker::isIdle() const
{
    return m_idle;
}

void Worker::flush()
{
    std::unique_lock<compat::Mutex> lock( m_lock );
    assert( m_paused == true || m_thread.get_id() == compat::Thread::id{} );
    m_idleCond.wait( lock, [this]() {
        return m_idle == true;
    });
    while ( m_tasks.empty() == false )
        m_tasks.pop();
    m_service->onFlushing();
}

void Worker::restart()
{
    m_service->onRestarted();
}

void Worker::mainloop()
{
    // It would be unsafe to call name() at the end of this function, since
    // we might stop the thread during ParserService destruction. This implies
    // that the underlying service has been deleted already.
    std::string serviceName = m_service->name();
    LOG_INFO("Entering ParserService [", serviceName, "] thread");
    setIdle( false );

    while ( true )
    {
        std::shared_ptr<Task> task;
        {
            std::unique_lock<compat::Mutex> lock( m_lock );
            if ( m_stopParser == true )
                break;
            if ( m_tasks.empty() == true || m_paused == true )
            {
                LOG_DEBUG( "Halting ParserService [", serviceName, "] mainloop" );
                setIdle( true );
                m_idleCond.notify_all();
                m_cond.wait( lock, [this]() {
                    return ( m_tasks.empty() == false && m_paused == false )
                            || m_stopParser == true;
                });
                LOG_DEBUG( "Resuming ParserService [", serviceName, "] mainloop" );
                // We might have been woken up because the parser is being destroyed
                if ( m_stopParser  == true )
                    break;
                setIdle( false );
            }
            // Otherwise it's safe to assume we have at least one element.
            LOG_DEBUG('[', serviceName, "] has ", m_tasks.size(), " tasks remaining" );
            task = std::move( m_tasks.front() );
            m_tasks.pop();
        }
        // Special case to restore uncompleted tasks from a parser thread
        if ( task == nullptr )
        {
            restoreTasks();
            continue;
        }
        if ( task->isStepCompleted( m_service->targetedStep() ) == true )
        {
            LOG_DEBUG( "Skipping completed task [", serviceName, "] on ", task->mrl() );
            m_parserCb->done( std::move( task ), Status::Success );
            continue;
        }
        Status status;
        try
        {
            LOG_DEBUG( "Executing ", serviceName, " task on ", task->mrl() );
            auto chrono = std::chrono::steady_clock::now();
            auto file = std::static_pointer_cast<File>( task->file() );
            auto media = std::static_pointer_cast<Media>( task->media() );

            if ( file != nullptr && file->isRemovable() )
            {
                auto folder = Folder::fetch( m_ml, file->folderId() );
                if ( folder->isPresent() == false )
                {
                    LOG_DEBUG( "Postponing parsing of ", file->rawMrl(),
                              " until the device containing it gets mounted back" );
                    m_parserCb->done( std::move( task ), Status::TemporaryUnavailable );
                    continue;
                }
            }
            task->startParserStep();
            status = m_service->run( *task );
            auto duration = std::chrono::steady_clock::now() - chrono;
            LOG_DEBUG( "Done executing ", serviceName, " task on ", task->mrl(), " in ",
                       std::chrono::duration_cast<std::chrono::milliseconds>( duration ).count(),
                       "ms. Result: ",
                       static_cast<std::underlying_type_t<parser::Status>>( status ) );
        }
        catch ( const fs::DeviceRemovedException& )
        {
            LOG_ERROR( "Parsing of ", task->mrl(), " was interrupted "
                       "due to its containing device being unmounted" );
            status = Status::TemporaryUnavailable;
        }
        catch ( const std::exception& ex )
        {
            LOG_ERROR( "Caught an exception during ", task->mrl(), " [", serviceName, "] parsing: ", ex.what() );
            status = Status::Fatal;
        }
        if ( handleServiceResult( *task, status ) == false )
            status = Status::Fatal;
        m_parserCb->done( std::move( task ), status );
    }
    LOG_INFO("Exiting ParserService [", serviceName, "] thread");
    setIdle( true );
}

void Worker::setIdle(bool isIdle)
{
    // Calling the idleChanged callback will trigger a call to isIdle, so set the value before
    // invoking it, otherwise we have an incoherent state.
    if ( m_idle != isIdle )
    {
        m_idle = isIdle;
        m_parserCb->onIdleChanged( isIdle );
    }
}

bool Worker::handleServiceResult( Task& task, Status status )
{
    if ( status == Status::Success )
    {
        task.markStepCompleted( m_service->targetedStep() );
        // We don't want to save the extraction step in database, as restarting a
        // task with extraction completed but analysis uncompleted wouldn't run
        // the extraction again, causing the analysis to run with no info.
        if ( m_service->targetedStep() != Step::MetadataExtraction )
            return task.saveParserStep();
        // We don't want to reset the entire retry count, as we would be stuck in
        // a "loop" in case the metadata analysis fails (we'd always reset the retry
        // count to zero, then fail, then run the extraction again, reset the retry,
        // fail the analysis, and so on.
        // We can't not increment the retry count for metadata extraction, since
        // in case a file makes (lib)VLC crash, we would always try again, and
        // therefor we would keep on crashing.
        // However we don't want to just increment the retry count, since it
        // would reach the maximum value too quickly (extraction would set retry
        // count to 1, analysis to 2, and in case of failure, next run would set
        // it over 3, while we only tried 2 times. Instead we just decrement it
        // when the extraction step succeeds
        return task.decrementRetryCount();
    }
    else if ( status == Status::Completed )
    {
        task.markStepCompleted( Step::Completed );
        return task.saveParserStep();
    }
    else if ( status == Status::Discarded )
    {
        return Task::destroy( m_ml, task.id() );
    }
    return true;
}

void Worker::restoreTasks()
{
    auto tasks = Task::fetchUncompleted( m_ml );
    if ( tasks.size() > 0 )
        LOG_INFO( "Resuming parsing on ", tasks.size(), " tasks" );
    else
        LOG_DEBUG( "No task to resume." );
    for ( auto& t : tasks )
    {
        {
            std::lock_guard<compat::Mutex> lock( m_lock );
            if ( m_stopParser == true )
                break;
        }

        if ( t->restoreLinkedEntities() == false )
            continue;
        m_parserCb->parse( std::move( t ) );
    }
}

}
}
<|endoftext|>


<commit_before>/*
 * SessionData.cpp
 *
 * Copyright (C) 2009-11 by RStudio, Inc.
 *
 * This program is licensed to you under the terms of version 3 of the
 * GNU Affero General Public License. This program is distributed WITHOUT
 * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
 * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
 *
 */

#include "SessionData.hpp"

#include <string>
#include <vector>
#include <sstream>

#include <boost/bind.hpp>
#include <boost/format.hpp>

#include <core/Log.hpp>
#include <core/Error.hpp>
#include <core/Exec.hpp>
#include <core/FileSerializer.hpp>
#include <core/StringUtils.hpp>

#define R_INTERNAL_FUNCTIONS
#include <r/RInternal.hpp>
#include <r/RSexp.hpp>
#include <r/RExec.hpp>
#include <r/RJson.hpp>
#include <r/ROptions.hpp>
#include <r/RFunctionHook.hpp>

#include <session/SessionModuleContext.hpp>

#include "SessionContentUrls.hpp"

using namespace core;

namespace session {
namespace modules {
namespace data {

namespace {

SEXP dataEntryHook(SEXP call, SEXP op, SEXP args, SEXP rho)
{
   // dataentry(data, modes)
   // see in_RX11_dataentry in modules/X11/dataentry.c for X11 impl
   // see Raqua_dataentry in modules/aqua/dataentry.c for aqua impl
   // see do_dataentry in gnuwin32/dataentry.c for win32 impl
   r::function_hook::checkArity(op, args, call);

   // currently not supported
   r::exec::errorCall(call, "Editing of matrix and data.frame objects is not "
                            "currently supported in RStudio");

   // NOTE: see implementation of dataViewerHook below for how to
   // properly merge exception/error strategies for the runtime
   // context of a hook

   return R_NilValue;
}

void appendTag(std::string* pHTML,
               const std::string& text,
               const std::string& tag,
               const std::string& className = std::string())
{
   std::string classAttrib;
   if (!className.empty())
      classAttrib = " class=\"" + className + "\"";

   boost::format fmt("<%1%%2%>%3%</%1%>");
   pHTML->append(boost::str(fmt % tag %
                                  classAttrib %
                                  string_utils::textToHtml(text)));
}

void appendTD(std::string* pHTML,
              const std::string& text,
              const std::string& className = std::string())
{
   appendTag(pHTML, text, "td", className);
}

void appendTH(std::string* pHTML, const std::string& text)
{
   appendTag(pHTML, text, "th");
}

SEXP dataViewerHook(SEXP call, SEXP op, SEXP args, SEXP rho)
{
   // dataviewer(x, title)
   // (see in_R_X11_dataviewer in modules/X11/dataentry.c for X11 impl)
   r::function_hook::checkArity(op, args, call);

   try
   {
      // validate title
      SEXP titleSEXP = CADR(args);
      if (!Rf_isString(titleSEXP) || Rf_length(titleSEXP) != 1)
         throw r::exec::RErrorException("invalid title argument");

      // validate data
      SEXP dataSEXP = CAR(args);
      if (TYPEOF(dataSEXP) != VECSXP)
         throw r::exec::RErrorException("invalid data argument (not a list)");

      // validate names (View ensures that length(names) == length(data))
      SEXP namesSEXP = Rf_getAttrib(dataSEXP, R_NamesSymbol);
      if (TYPEOF(namesSEXP) != STRSXP ||
          Rf_length(namesSEXP) != Rf_length(dataSEXP))
      {
         throw r::exec::RErrorException(
                              "invalid data argument (names not specified)");
      }

      // extract title and column names
      std::string title = r::sexp::asString(titleSEXP);
      std::vector<std::string> columnNames;
      Error error = r::sexp::extract(namesSEXP, &columnNames);
      if (error)
         throw r::exec::RErrorException("invalid names: " +
                                        error.code().message());

      // get column lenghts and then calculate # of rows based on the maximum #
      // of elements in single column (technically R can pass columns which have
      // a disparate # of rows to this method)
      int columnCount = r::sexp::length(dataSEXP);
      int rowCount = 0;
      std::vector<int> columnLengths;
      for (int i=0; i<columnCount; i++)
      {
          // get the column and record its length (updating rowCount)
          SEXP columnSEXP = VECTOR_ELT(dataSEXP, i);
          int columnLength = r::sexp::length(columnSEXP);
          columnLengths.push_back(columnLength);
          rowCount = std::max(columnLength, rowCount);

          // validate data type (R converts all inbound vectors to REAL or STR)
          if (TYPEOF(columnSEXP) != REALSXP && TYPEOF(columnSEXP) != STRSXP)
          {
             throw r::exec::RErrorException("Unexpected type for column: "
                                            + r::sexp::typeAsString(columnSEXP));
          }
      }

      // calculate rows to display
      const int kMaxRows = 1000;
      int displayedRows = std::min(rowCount, kMaxRows);

      // format the data for presentation
      r::sexp::Protect rProtect;
      SEXP formattedDataSEXP = Rf_allocVector(VECSXP, columnCount);
      rProtect.add(formattedDataSEXP);
      for (int i=0; i<columnCount; i++)
      {
         SEXP columnSEXP = VECTOR_ELT(dataSEXP, i);
         SEXP formattedColumnSEXP;
         r::exec::RFunction formatFx(".rs.formatDataColumn");
         formatFx.addParam(columnSEXP);
         formatFx.addParam(displayedRows);
         Error error = formatFx.call(&formattedColumnSEXP, &rProtect);
         if (error)
            throw r::exec::RErrorException(error.summary());
         SET_VECTOR_ELT(formattedDataSEXP, i, formattedColumnSEXP);
       }

      // write html header
      boost::format headerFmt(
         "\n"
         "  \n"
         "     <title>%1%</title>\n"
         "     <meta charset=\"utf-8\"/>\n"
         "     <link rel=\"stylesheet\" type=\"text/css\" href=\"../css/data.css\"/>\n"
         "  \n"
         "  \n");
      std::string html = boost::str(headerFmt % title);

      // output begin table & header
      html += "
\n";
      html += "

\n";
      html += " 	"; // above row numbers
      std::for_each(columnNames.begin(),
                    columnNames.end(),
                    boost::bind(appendTH, &html, _1));
      html += "\n

\n";

      html += "
\n";
      // output rows
      for (int row=0; row<displayedRows; row++)
      {
         html += "
\n";

         // row number
         appendTD(&html, boost::lexical_cast<std::string>(row+1), "rn");

         // output a data element from each column where this row is available
         for (int col=0; col<Rf_length(formattedDataSEXP); col++)
         {
            if (columnLengths[col] > row)
            {
               SEXP columnSEXP = VECTOR_ELT(formattedDataSEXP, col);
               SEXP stringSEXP = STRING_ELT(columnSEXP, row);
               if (stringSEXP != NA_STRING && r::sexp::length(stringSEXP) > 0)
               {
                  std::string text(Rf_translateChar(stringSEXP));
                  appendTD(&html, text);
               }
               else
               {
                  html += " 	";
               }
            }
         }

         html += "\n
\n";
      }
      html += "
\n";

      // append table footer
      html += "\n
\n";

      // append document footer
      html += "\n";

      // fire show data event
      json::Object dataItem;
      dataItem["title"] = title;
      dataItem["totalObservations"] = rowCount;
      dataItem["displayedObservations"] = displayedRows;
      dataItem["variables"] = columnCount;
      dataItem["contentUrl"] = content_urls::provision(title, html, ".htm");
      ClientEvent event(client_events::kShowData, dataItem);
      module_context::enqueClientEvent(event);

      // done
      return R_NilValue;
   }
   catch(r::exec::RErrorException& e)
   {
      r::exec::errorCall(call, e.message());
   }
   CATCH_UNEXPECTED_EXCEPTION

   // keep compiler happy
   return R_NilValue;
}

} // anonymous namespace

Error initialize()
{
   using boost::bind;
   using namespace r::function_hook ;
   using namespace session::module_context;
   ExecBlock initBlock ;
   initBlock.addFunctions()
      (bind(registerReplaceHook, "dataentry", dataEntryHook, (CCODE*)NULL))
      (bind(registerReplaceHook, "dataviewer", dataViewerHook,(CCODE*)NULL))
      (bind(sourceModuleRFile, "SessionData.R"));

   return initBlock.execute();
}

} // namespace data
} // namespace modules
} // namesapce session

<commit_msg>clip viewed datasets at 100 columns so the client DOM doesn't get overwhelmed by massive numbers of columns<commit_after>/*
 * SessionData.cpp
 *
 * Copyright (C) 2009-11 by RStudio, Inc.
 *
 * This program is licensed to you under the terms of version 3 of the
 * GNU Affero General Public License. This program is distributed WITHOUT
 * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
 * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
 *
 */

#include "SessionData.hpp"

#include <string>
#include <vector>
#include <sstream>

#include <boost/bind.hpp>
#include <boost/format.hpp>

#include <core/Log.hpp>
#include <core/Error.hpp>
#include <core/Exec.hpp>
#include <core/FileSerializer.hpp>
#include <core/StringUtils.hpp>

#define R_INTERNAL_FUNCTIONS
#include <r/RInternal.hpp>
#include <r/RSexp.hpp>
#include <r/RExec.hpp>
#include <r/RJson.hpp>
#include <r/ROptions.hpp>
#include <r/RFunctionHook.hpp>

#include <session/SessionModuleContext.hpp>

#include "SessionContentUrls.hpp"

using namespace core;

namespace session {
namespace modules {
namespace data {

namespace {

SEXP dataEntryHook(SEXP call, SEXP op, SEXP args, SEXP rho)
{
   // dataentry(data, modes)
   // see in_RX11_dataentry in modules/X11/dataentry.c for X11 impl
   // see Raqua_dataentry in modules/aqua/dataentry.c for aqua impl
   // see do_dataentry in gnuwin32/dataentry.c for win32 impl
   r::function_hook::checkArity(op, args, call);

   // currently not supported
   r::exec::errorCall(call, "Editing of matrix and data.frame objects is not "
                            "currently supported in RStudio");

   // NOTE: see implementation of dataViewerHook below for how to
   // properly merge exception/error strategies for the runtime
   // context of a hook

   return R_NilValue;
}

void appendTag(std::string* pHTML,
               const std::string& text,
               const std::string& tag,
               const std::string& className = std::string())
{
   std::string classAttrib;
   if (!className.empty())
      classAttrib = " class=\"" + className + "\"";

   boost::format fmt("<%1%%2%>%3%</%1%>");
   pHTML->append(boost::str(fmt % tag %
                                  classAttrib %
                                  string_utils::textToHtml(text)));
}

void appendTD(std::string* pHTML,
              const std::string& text,
              const std::string& className = std::string())
{
   appendTag(pHTML, text, "td", className);
}

void appendTH(std::string* pHTML, const std::string& text)
{
   appendTag(pHTML, text, "th");
}

SEXP dataViewerHook(SEXP call, SEXP op, SEXP args, SEXP rho)
{
   // dataviewer(x, title)
   // (see in_R_X11_dataviewer in modules/X11/dataentry.c for X11 impl)
   r::function_hook::checkArity(op, args, call);

   try
   {
      // validate title
      SEXP titleSEXP = CADR(args);
      if (!Rf_isString(titleSEXP) || Rf_length(titleSEXP) != 1)
         throw r::exec::RErrorException("invalid title argument");

      // validate data
      SEXP dataSEXP = CAR(args);
      if (TYPEOF(dataSEXP) != VECSXP)
         throw r::exec::RErrorException("invalid data argument (not a list)");

      // validate names (View ensures that length(names) == length(data))
      SEXP namesSEXP = Rf_getAttrib(dataSEXP, R_NamesSymbol);
      if (TYPEOF(namesSEXP) != STRSXP ||
          Rf_length(namesSEXP) != Rf_length(dataSEXP))
      {
         throw r::exec::RErrorException(
                              "invalid data argument (names not specified)");
      }

      // get column count
      int columnCount = r::sexp::length(dataSEXP);

      // calculate columns to display
      const int kMaxColumns = 100;
      int displayedColumns = std::min(columnCount, kMaxColumns);

      // extract title and column names
      std::string title = r::sexp::asString(titleSEXP);
      std::vector<std::string> columnNames;
      Error error = r::sexp::extract(namesSEXP, &columnNames);
      if (error)
         throw r::exec::RErrorException("invalid names: " +
                                        error.code().message());

      // truncate columns names to displayedColumns
      columnNames.resize(displayedColumns);

      // get column lenghts and then calculate # of rows based on the maximum #
      // of elements in single column (technically R can pass columns which have
      // a disparate # of rows to this method)
      int rowCount = 0;
      std::vector<int> columnLengths;
      for (int i=0; i<displayedColumns; i++)
      {
          // get the column and record its length (updating rowCount)
          SEXP columnSEXP = VECTOR_ELT(dataSEXP, i);
          int columnLength = r::sexp::length(columnSEXP);
          columnLengths.push_back(columnLength);
          rowCount = std::max(columnLength, rowCount);

          // validate data type (R converts all inbound vectors to REAL or STR)
          if (TYPEOF(columnSEXP) != REALSXP && TYPEOF(columnSEXP) != STRSXP)
          {
             throw r::exec::RErrorException("Unexpected type for column: "
                                            + r::sexp::typeAsString(columnSEXP));
          }
      }

      // calculate rows to display
      const int kMaxRows = 1000;
      int displayedRows = std::min(rowCount, kMaxRows);

      // format the data for presentation
      r::sexp::Protect rProtect;
      SEXP formattedDataSEXP = Rf_allocVector(VECSXP, displayedColumns);
      rProtect.add(formattedDataSEXP);
      for (int i=0; i<displayedColumns; i++)
      {
         SEXP columnSEXP = VECTOR_ELT(dataSEXP, i);
         SEXP formattedColumnSEXP;
         r::exec::RFunction formatFx(".rs.formatDataColumn");
         formatFx.addParam(columnSEXP);
         formatFx.addParam(displayedRows);
         Error error = formatFx.call(&formattedColumnSEXP, &rProtect);
         if (error)
            throw r::exec::RErrorException(error.summary());
         SET_VECTOR_ELT(formattedDataSEXP, i, formattedColumnSEXP);
       }

      // write html header
      boost::format headerFmt(
         "\n"
         "  \n"
         "     <title>%1%</title>\n"
         "     <meta charset=\"utf-8\"/>\n"
         "     <link rel=\"stylesheet\" type=\"text/css\" href=\"../css/data.css\"/>\n"
         "  \n"
         "  \n");
      std::string html = boost::str(headerFmt % title);

      // output begin table & header
      html += "
\n";
      html += "

\n";
      html += " 	"; // above row numbers
      std::for_each(columnNames.begin(),
                    columnNames.end(),
                    boost::bind(appendTH, &html, _1));
      html += "\n

\n";

      html += "
\n";
      // output rows
      for (int row=0; row<displayedRows; row++)
      {
         html += "
\n";

         // row number
         appendTD(&html, boost::lexical_cast<std::string>(row+1), "rn");

         // output a data element from each column where this row is available
         for (int col=0; col<Rf_length(formattedDataSEXP); col++)
         {
            if (columnLengths[col] > row)
            {
               SEXP columnSEXP = VECTOR_ELT(formattedDataSEXP, col);
               SEXP stringSEXP = STRING_ELT(columnSEXP, row);
               if (stringSEXP != NA_STRING && r::sexp::length(stringSEXP) > 0)
               {
                  std::string text(Rf_translateChar(stringSEXP));
                  appendTD(&html, text);
               }
               else
               {
                  html += " 	";
               }
            }
         }

         html += "\n
\n";
      }
      html += "
\n";

      // append table footer
      html += "\n
\n";

      // append document footer
      html += "\n";

      // fire show data event
      json::Object dataItem;
      dataItem["title"] = title;
      dataItem["totalObservations"] = rowCount;
      dataItem["displayedObservations"] = displayedRows;
      dataItem["variables"] = columnCount;
      dataItem["displayedVariables"] = displayedColumns;
      dataItem["contentUrl"] = content_urls::provision(title, html, ".htm");
      ClientEvent event(client_events::kShowData, dataItem);
      module_context::enqueClientEvent(event);

      // done
      return R_NilValue;
   }
   catch(r::exec::RErrorException& e)
   {
      r::exec::errorCall(call, e.message());
   }
   CATCH_UNEXPECTED_EXCEPTION

   // keep compiler happy
   return R_NilValue;
}

} // anonymous namespace

Error initialize()
{
   using boost::bind;
   using namespace r::function_hook ;
   using namespace session::module_context;
   ExecBlock initBlock ;
   initBlock.addFunctions()
      (bind(registerReplaceHook, "dataentry", dataEntryHook, (CCODE*)NULL))
      (bind(registerReplaceHook, "dataviewer", dataViewerHook,(CCODE*)NULL))
      (bind(sourceModuleRFile, "SessionData.R"));

   return initBlock.execute();
}

} // namespace data
} // namespace modules
} // namesapce session

<|endoftext|>


<commit_before>// Copyright 2014, University of Freiburg,
// Chair of Algorithms and Data Structures.
// Author: Björn Buchhold (buchhold@informatik.uni-freiburg.de)

#include "./SparqlParser.h"
#include "./ParseException.h"
#include "../util/StringUtils.h"

// _____________________________________________________________________________
ParsedQuery SparqlParser::parse(const string& query) {
  ParsedQuery result;

  // Split prologue
  size_t i = query.find("SELECT");
  if (i == string::npos) {
    throw ParseException("Missing keyword \"SELECT\", "
        "currently only select queries are supported.");
  }

  size_t j = query.find("WHERE");
  if (j == string::npos) {
    throw ParseException("Missing keyword \"WHERE\", "
        "currently only select queries are supported.");
  }

  if (i >= j) {
    throw ParseException("Keyword \"WHERE\" "
        "found after keyword \"SELECT\". Invalid query.");
  }

  parsePrologue(ad_utility::strip(query.substr(0, i), " \n\t"), result);
  parseSelect(ad_utility::strip(query.substr(i, j - (i + 1)), " \n\t"), result);
  parseWhere(ad_utility::strip(query.substr(j), " \n\t"), result);

  return result;
}

// _____________________________________________________________________________
void SparqlParser::parsePrologue(string str, ParsedQuery& query) {
  if (str.find("BASE") != string::npos) {
    throw ParseException("Bases are not supported, yet.");
  }

  if (str.find("BASE") != string::npos) {
    throw ParseException("Bases are not supported, yet.");
  }

  size_t i = str.find("PREFIX");
  while (i != string::npos) {
    size_t j = str.find("PREFIX", i + 1);
    addPrefix(str.substr(i, j - i), query);
    i = j;
  }
}

// _____________________________________________________________________________
void SparqlParser::addPrefix(const string& str, ParsedQuery& query) {
  auto parts = ad_utility::split(ad_utility::strip(str, ' '), ' ');
  if (parts.size() != 3) {
    throw ParseException(string("Invalid PREFIX statement: ") + str);
  }
  string uri = ad_utility::strip(parts[2], " \t\n");
  if (uri.size() == 0 || uri[0] != '<' || uri[uri.size() - 1] != '>') {
    throw ParseException(string("Invalid URI in PREFIX: ") + uri);
  }
  SparqlPrefix p{ad_utility::strip(parts[1], " :\t\n"), uri};
  query._prefixes.emplace_back(p);
};

// _____________________________________________________________________________
void SparqlParser::parseSelect(const string& str, ParsedQuery& query) {
  assert(ad_utility::startsWith(str, "SELECT"));
  auto vars = ad_utility::split(str, ' ');
  for (size_t i = 1; i < vars.size(); ++i) {
    if (ad_utility::startsWith(vars[i], "?")) {
      query._selectedVariables.push_back(vars[i]);
    } else {
      throw ParseException(string("Invalid variable in select clause: \"") +
          vars[i] + "\"");
    }
  }
}

// _____________________________________________________________________________
void SparqlParser::parseWhere(const string& str, ParsedQuery& query) {
  size_t i = str.find('{');
  size_t j = str.find('}', i);
  if (i == string::npos) {
    throw ParseException("Need curly braces in where clause.");
  }
  string inner = ad_utility::strip(str.substr(i + 1, j - i - 1), "\n\t ");

  // Split where clauses. Cannot simply split at dots, because they may occur
  // in URIs.
  vector<string> clauses;
  size_t start = 0;
  bool insideUri = false;
  while (start < inner.size()) {
    size_t k = start;
    while (k < inner.size()) {
      if (!insideUri) {
        if (inner[k] == '.') {
          clauses.emplace_back(inner.substr(start, k - start));
          break;
        }
        if (inner[k] == '<') {insideUri = true;}
      } else {
        if (inner[k] == '>') {insideUri = false;}
      }
      ++k;
    }
    if (k == inner.size()) {clauses.emplace_back(inner.substr(start));}
    start = k + 1;
  }
  for (const string& clause: clauses) {
    string c = ad_utility::strip(clause, ' ');
    if (c.size() > 0) {
      addWhereTriple(c, query);
    }
  }
}

// _____________________________________________________________________________
void SparqlParser::addWhereTriple(const string& str, ParsedQuery& query) {
  auto spo = ad_utility::split(str, ' ');
  if (spo.size() != 3) {
    throw ParseException(string("Invalid triple, expected format: \"s p o\". "
        "Triple was: ") + str);
  }
  query._whereClauseTriples.emplace_back(SparqlTriple{spo[0], spo[1], spo[2]});
}
<commit_msg>Fixed another problem with separator chars in queries<commit_after>// Copyright 2014, University of Freiburg,
// Chair of Algorithms and Data Structures.
// Author: Björn Buchhold (buchhold@informatik.uni-freiburg.de)

#include <bits/algorithmfwd.h>
#include "./SparqlParser.h"
#include "./ParseException.h"
#include "../util/StringUtils.h"

// _____________________________________________________________________________
ParsedQuery SparqlParser::parse(const string& query) {
  ParsedQuery result;

  // Split prologue
  size_t i = query.find("SELECT");
  if (i == string::npos) {
    throw ParseException("Missing keyword \"SELECT\", "
        "currently only select queries are supported.");
  }

  size_t j = query.find("WHERE");
  if (j == string::npos) {
    throw ParseException("Missing keyword \"WHERE\", "
        "currently only select queries are supported.");
  }

  if (i >= j) {
    throw ParseException("Keyword \"WHERE\" "
        "found after keyword \"SELECT\". Invalid query.");
  }

  parsePrologue(ad_utility::strip(query.substr(0, i), " \n\t"), result);
  parseSelect(ad_utility::strip(query.substr(i, j - (i + 1)), " \n\t"), result);
  parseWhere(ad_utility::strip(query.substr(j), " \n\t"), result);

  return result;
}

// _____________________________________________________________________________
void SparqlParser::parsePrologue(string str, ParsedQuery& query) {
  if (str.find("BASE") != string::npos) {
    throw ParseException("Bases are not supported, yet.");
  }

  if (str.find("BASE") != string::npos) {
    throw ParseException("Bases are not supported, yet.");
  }

  size_t i = str.find("PREFIX");
  while (i != string::npos) {
    size_t j = str.find("PREFIX", i + 1);
    addPrefix(str.substr(i, j - i), query);
    i = j;
  }
}

// _____________________________________________________________________________
void SparqlParser::addPrefix(const string& str, ParsedQuery& query) {
  auto parts = ad_utility::split(ad_utility::strip(str, ' '), ' ');
  if (parts.size() != 3) {
    throw ParseException(string("Invalid PREFIX statement: ") + str);
  }
  string uri = ad_utility::strip(parts[2], " \t\n");
  if (uri.size() == 0 || uri[0] != '<' || uri[uri.size() - 1] != '>') {
    throw ParseException(string("Invalid URI in PREFIX: ") + uri);
  }
  SparqlPrefix p{ad_utility::strip(parts[1], " :\t\n"), uri};
  query._prefixes.emplace_back(p);
};

// _____________________________________________________________________________
void SparqlParser::parseSelect(const string& str, ParsedQuery& query) {
  assert(ad_utility::startsWith(str, "SELECT"));
  auto vars = ad_utility::split(str, ' ');
  for (size_t i = 1; i < vars.size(); ++i) {
    if (ad_utility::startsWith(vars[i], "?")) {
      query._selectedVariables.push_back(vars[i]);
    } else {
      throw ParseException(string("Invalid variable in select clause: \"") +
          vars[i] + "\"");
    }
  }
}

// _____________________________________________________________________________
void SparqlParser::parseWhere(const string& str, ParsedQuery& query) {
  size_t i = str.find('{');
  size_t j = str.find('}', i);
  if (i == string::npos) {
    throw ParseException("Need curly braces in where clause.");
  }
  string inner = ad_utility::strip(str.substr(i + 1, j - i - 1), "\n\t ");

  // Split where clauses. Cannot simply split at dots, because they may occur
  // in URIs.
  vector<string> clauses;
  size_t start = 0;
  bool insideUri = false;
  while (start < inner.size()) {
    size_t k = start;
    while (k < inner.size()) {
      if (!insideUri) {
        if (inner[k] == '.') {
          clauses.emplace_back(inner.substr(start, k - start));
          break;
        }
        if (inner[k] == '<') {insideUri = true;}
      } else {
        if (inner[k] == '>') {insideUri = false;}
      }
      ++k;
    }
    if (k == inner.size()) {clauses.emplace_back(inner.substr(start));}
    start = k + 1;
  }
  for (const string& clause: clauses) {
    string c = ad_utility::strip(clause, ' ');
    if (c.size() > 0) {
      addWhereTriple(c, query);
    }
  }
}

// _____________________________________________________________________________
void SparqlParser::addWhereTriple(const string& str, ParsedQuery& query) {
  auto spo = ad_utility::split(str, ' ');
  if (spo.size() != 3) {
    throw ParseException(string("Invalid triple, expected format: \"s p o\". "
        "Triple was: ") + str);
  }
  query._whereClauseTriples.emplace_back(SparqlTriple{
      ad_utility::strip(spo[0], "\n\t "),
      ad_utility::strip(spo[1], "\n\t "),
      ad_utility::strip(spo[2], "\n\t ")
  });
}
<|endoftext|>


<commit_before>// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/bind.h"
#include "base/command_line.h"
#include "base/run_loop.h"
#include "base/thread_task_runner_handle.h"
#include "base/trace_event/memory_dump_manager.h"
#include "base/trace_event/memory_dump_provider.h"
#include "base/trace_event/memory_dump_request_args.h"
#include "content/public/browser/tracing_controller.h"
#include "content/public/common/content_switches.h"
#include "content/public/test/browser_test_utils.h"
#include "content/public/test/content_browser_test.h"
#include "content/public/test/content_browser_test_utils.h"
#include "content/shell/browser/shell.h"
#include "testing/gmock/include/gmock/gmock.h"

using base::trace_event::MemoryDumpArgs;
using base::trace_event::MemoryDumpManager;
using base::trace_event::MemoryDumpType;
using base::trace_event::ProcessMemoryDump;
using testing::_;
using testing::Return;

namespace content {

// A mock dump provider, used to check that dump requests actually end up
// creating memory dumps.
class MockDumpProvider : public base::trace_event::MemoryDumpProvider {
 public:
  MOCK_METHOD2(OnMemoryDump, bool(const MemoryDumpArgs& args,
                                  ProcessMemoryDump* pmd));
};

class MemoryTracingTest : public ContentBrowserTest {
 public:
  void DoRequestGlobalDump(const base::trace_event::MemoryDumpCallback& cb) {
    MemoryDumpArgs dump_args = { MemoryDumpArgs::LevelOfDetail::HIGH };
    MemoryDumpManager::GetInstance()->RequestGlobalDump(
        MemoryDumpType::EXPLICITLY_TRIGGERED, dump_args, cb);
  }

  // Used as callback argument for MemoryDumpManager::RequestGlobalDump():
  void OnGlobalMemoryDumpDone(
      scoped_refptr<base::SingleThreadTaskRunner> task_runner,
      base::Closure closure,
      uint64 dump_guid,
      bool success) {
    // Make sure we run the RunLoop closure on the same thread that originated
    // the run loop (which is the IN_PROC_BROWSER_TEST_F main thread).
    if (!task_runner->RunsTasksOnCurrentThread()) {
      task_runner->PostTask(
          FROM_HERE, base::Bind(&MemoryTracingTest::OnGlobalMemoryDumpDone,
                                base::Unretained(this), task_runner, closure,
                                dump_guid, success));
      return;
    }
    ++callback_call_count_;
    last_callback_dump_guid_ = dump_guid;
    last_callback_success_ = success;
    closure.Run();
  }

 protected:
  void SetUp() override {
    callback_call_count_ = 0;
    last_callback_dump_guid_ = 0;
    last_callback_success_ = false;
    MemoryDumpManager::GetInstance()->Initialize();
    mock_dump_provider_.reset(new MockDumpProvider());
    MemoryDumpManager::GetInstance()->RegisterDumpProvider(
        mock_dump_provider_.get());
    // TODO(primiano): This should be done via TraceConfig.
    // See https://goo.gl/5Hj3o0.
    MemoryDumpManager::GetInstance()->DisablePeriodicDumpsForTesting();
    ContentBrowserTest::SetUp();
  }

  void TearDown() override {
    MemoryDumpManager::GetInstance()->UnregisterDumpProvider(
        mock_dump_provider_.get());
    mock_dump_provider_.reset();
    ContentBrowserTest::TearDown();
  }

  void EnableMemoryTracing() {
    base::RunLoop run_loop;
    std::string category_filter = MemoryDumpManager::kTraceCategoryForTesting;
    base::trace_event::TraceConfig trace_config(category_filter, "");
    bool success = TracingController::GetInstance()->EnableRecording(
      trace_config, run_loop.QuitClosure());
    EXPECT_TRUE(success);
    run_loop.Run();
  }

  void DisableTracing() {
    bool success = TracingController::GetInstance()->DisableRecording(NULL);
    EXPECT_TRUE(success);
    base::RunLoop().RunUntilIdle();
  }

  void RequestGlobalDumpAndWait(bool from_renderer_thread) {
    base::RunLoop run_loop;
    base::trace_event::MemoryDumpCallback callback = base::Bind(
        &MemoryTracingTest::OnGlobalMemoryDumpDone, base::Unretained(this),
        base::ThreadTaskRunnerHandle::Get(), run_loop.QuitClosure());
    if (from_renderer_thread) {
      PostTaskToInProcessRendererAndWait(
          base::Bind(&MemoryTracingTest::DoRequestGlobalDump,
                     base::Unretained(this), callback));
    } else {
      DoRequestGlobalDump(callback);
    }
    run_loop.Run();
  }

  void Navigate(Shell* shell) {
    NavigateToURL(shell, GetTestUrl("", "title.html"));
  }

  base::Closure on_memory_dump_complete_closure_;
  scoped_ptr<MockDumpProvider> mock_dump_provider_;
  uint32 callback_call_count_;
  uint64 last_callback_dump_guid_;
  bool last_callback_success_;
};

class SingleProcessMemoryTracingTest : public MemoryTracingTest {
 public:
  SingleProcessMemoryTracingTest() {}

  void SetUpCommandLine(base::CommandLine* command_line) override {
    command_line->AppendSwitch(switches::kSingleProcess);
  }
};

// Checks that a memory dump initiated from a the main browser thread ends up in
// a single dump even in single process mode.
IN_PROC_BROWSER_TEST_F(SingleProcessMemoryTracingTest,
                       BrowserInitiatedSingleDump) {
  Navigate(shell());

  EXPECT_CALL(*mock_dump_provider_, OnMemoryDump(_,_)).WillOnce(Return(true));

  EnableMemoryTracing();
  RequestGlobalDumpAndWait(false /* from_renderer_thread */);
  EXPECT_EQ(1u, callback_call_count_);
  EXPECT_NE(0u, last_callback_dump_guid_);
  EXPECT_TRUE(last_callback_success_);
  DisableTracing();
}

// Checks that a memory dump initiated from a renderer thread ends up in a
// single dump even in single process mode.
IN_PROC_BROWSER_TEST_F(SingleProcessMemoryTracingTest,
                       RendererInitiatedSingleDump) {
  Navigate(shell());

  EXPECT_CALL(*mock_dump_provider_, OnMemoryDump(_,_)).WillOnce(Return(true));

  EnableMemoryTracing();
  RequestGlobalDumpAndWait(true /* from_renderer_thread */);
  EXPECT_EQ(1u, callback_call_count_);
  EXPECT_NE(0u, last_callback_dump_guid_);
  EXPECT_TRUE(last_callback_success_);
  DisableTracing();
}

IN_PROC_BROWSER_TEST_F(SingleProcessMemoryTracingTest, ManyInterleavedDumps) {
  Navigate(shell());

  EXPECT_CALL(*mock_dump_provider_, OnMemoryDump(_,_))
      .Times(4)
      .WillRepeatedly(Return(true));

  EnableMemoryTracing();

  RequestGlobalDumpAndWait(true /* from_renderer_thread */);
  EXPECT_NE(0u, last_callback_dump_guid_);
  EXPECT_TRUE(last_callback_success_);

  RequestGlobalDumpAndWait(false /* from_renderer_thread */);
  EXPECT_NE(0u, last_callback_dump_guid_);
  EXPECT_TRUE(last_callback_success_);

  RequestGlobalDumpAndWait(false /* from_renderer_thread */);
  EXPECT_NE(0u, last_callback_dump_guid_);
  EXPECT_TRUE(last_callback_success_);

  RequestGlobalDumpAndWait(true /* from_renderer_thread */);
  EXPECT_EQ(4u, callback_call_count_);
  EXPECT_NE(0u, last_callback_dump_guid_);
  EXPECT_TRUE(last_callback_success_);

  DisableTracing();
}

// Checks that a memory dump initiated from a the main browser thread ends up in
// a successful dump.
IN_PROC_BROWSER_TEST_F(MemoryTracingTest, BrowserInitiatedDump) {
  Navigate(shell());

  EXPECT_CALL(*mock_dump_provider_, OnMemoryDump(_,_)).WillOnce(Return(true));

  EnableMemoryTracing();
  RequestGlobalDumpAndWait(false /* from_renderer_thread */);
  EXPECT_EQ(1u, callback_call_count_);
  EXPECT_NE(0u, last_callback_dump_guid_);
  EXPECT_TRUE(last_callback_success_);
  DisableTracing();
}

}  // namespace content
<commit_msg>Disable flaky SingleProcessMemoryTracingTest tests.<commit_after>// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/bind.h"
#include "base/command_line.h"
#include "base/run_loop.h"
#include "base/thread_task_runner_handle.h"
#include "base/trace_event/memory_dump_manager.h"
#include "base/trace_event/memory_dump_provider.h"
#include "base/trace_event/memory_dump_request_args.h"
#include "content/public/browser/tracing_controller.h"
#include "content/public/common/content_switches.h"
#include "content/public/test/browser_test_utils.h"
#include "content/public/test/content_browser_test.h"
#include "content/public/test/content_browser_test_utils.h"
#include "content/shell/browser/shell.h"
#include "testing/gmock/include/gmock/gmock.h"

using base::trace_event::MemoryDumpArgs;
using base::trace_event::MemoryDumpManager;
using base::trace_event::MemoryDumpType;
using base::trace_event::ProcessMemoryDump;
using testing::_;
using testing::Return;

namespace content {

// A mock dump provider, used to check that dump requests actually end up
// creating memory dumps.
class MockDumpProvider : public base::trace_event::MemoryDumpProvider {
 public:
  MOCK_METHOD2(OnMemoryDump, bool(const MemoryDumpArgs& args,
                                  ProcessMemoryDump* pmd));
};

class MemoryTracingTest : public ContentBrowserTest {
 public:
  void DoRequestGlobalDump(const base::trace_event::MemoryDumpCallback& cb) {
    MemoryDumpArgs dump_args = { MemoryDumpArgs::LevelOfDetail::HIGH };
    MemoryDumpManager::GetInstance()->RequestGlobalDump(
        MemoryDumpType::EXPLICITLY_TRIGGERED, dump_args, cb);
  }

  // Used as callback argument for MemoryDumpManager::RequestGlobalDump():
  void OnGlobalMemoryDumpDone(
      scoped_refptr<base::SingleThreadTaskRunner> task_runner,
      base::Closure closure,
      uint64 dump_guid,
      bool success) {
    // Make sure we run the RunLoop closure on the same thread that originated
    // the run loop (which is the IN_PROC_BROWSER_TEST_F main thread).
    if (!task_runner->RunsTasksOnCurrentThread()) {
      task_runner->PostTask(
          FROM_HERE, base::Bind(&MemoryTracingTest::OnGlobalMemoryDumpDone,
                                base::Unretained(this), task_runner, closure,
                                dump_guid, success));
      return;
    }
    ++callback_call_count_;
    last_callback_dump_guid_ = dump_guid;
    last_callback_success_ = success;
    closure.Run();
  }

 protected:
  void SetUp() override {
    callback_call_count_ = 0;
    last_callback_dump_guid_ = 0;
    last_callback_success_ = false;
    MemoryDumpManager::GetInstance()->Initialize();
    mock_dump_provider_.reset(new MockDumpProvider());
    MemoryDumpManager::GetInstance()->RegisterDumpProvider(
        mock_dump_provider_.get());
    // TODO(primiano): This should be done via TraceConfig.
    // See https://goo.gl/5Hj3o0.
    MemoryDumpManager::GetInstance()->DisablePeriodicDumpsForTesting();
    ContentBrowserTest::SetUp();
  }

  void TearDown() override {
    MemoryDumpManager::GetInstance()->UnregisterDumpProvider(
        mock_dump_provider_.get());
    mock_dump_provider_.reset();
    ContentBrowserTest::TearDown();
  }

  void EnableMemoryTracing() {
    base::RunLoop run_loop;
    std::string category_filter = MemoryDumpManager::kTraceCategoryForTesting;
    base::trace_event::TraceConfig trace_config(category_filter, "");
    bool success = TracingController::GetInstance()->EnableRecording(
      trace_config, run_loop.QuitClosure());
    EXPECT_TRUE(success);
    run_loop.Run();
  }

  void DisableTracing() {
    bool success = TracingController::GetInstance()->DisableRecording(NULL);
    EXPECT_TRUE(success);
    base::RunLoop().RunUntilIdle();
  }

  void RequestGlobalDumpAndWait(bool from_renderer_thread) {
    base::RunLoop run_loop;
    base::trace_event::MemoryDumpCallback callback = base::Bind(
        &MemoryTracingTest::OnGlobalMemoryDumpDone, base::Unretained(this),
        base::ThreadTaskRunnerHandle::Get(), run_loop.QuitClosure());
    if (from_renderer_thread) {
      PostTaskToInProcessRendererAndWait(
          base::Bind(&MemoryTracingTest::DoRequestGlobalDump,
                     base::Unretained(this), callback));
    } else {
      DoRequestGlobalDump(callback);
    }
    run_loop.Run();
  }

  void Navigate(Shell* shell) {
    NavigateToURL(shell, GetTestUrl("", "title.html"));
  }

  base::Closure on_memory_dump_complete_closure_;
  scoped_ptr<MockDumpProvider> mock_dump_provider_;
  uint32 callback_call_count_;
  uint64 last_callback_dump_guid_;
  bool last_callback_success_;
};

class SingleProcessMemoryTracingTest : public MemoryTracingTest {
 public:
  SingleProcessMemoryTracingTest() {}

  void SetUpCommandLine(base::CommandLine* command_line) override {
    command_line->AppendSwitch(switches::kSingleProcess);
  }
};

// This test is flaky on Mac. See crbug.com/522009.
#if defined(OS_MACOSX)
#define MAYBE_BrowserInitiatedSingleDump DISABLED_BrowserInitiatedSingleDump
#else
#define MAYBE_BrowserInitiatedSingleDump BrowserInitiatedSingleDump
#endif
// Checks that a memory dump initiated from a the main browser thread ends up in
// a single dump even in single process mode.
IN_PROC_BROWSER_TEST_F(SingleProcessMemoryTracingTest,
                       MAYBE_BrowserInitiatedSingleDump) {
  Navigate(shell());

  EXPECT_CALL(*mock_dump_provider_, OnMemoryDump(_,_)).WillOnce(Return(true));

  EnableMemoryTracing();
  RequestGlobalDumpAndWait(false /* from_renderer_thread */);
  EXPECT_EQ(1u, callback_call_count_);
  EXPECT_NE(0u, last_callback_dump_guid_);
  EXPECT_TRUE(last_callback_success_);
  DisableTracing();
}

// Checks that a memory dump initiated from a renderer thread ends up in a
// single dump even in single process mode.
IN_PROC_BROWSER_TEST_F(SingleProcessMemoryTracingTest,
                       RendererInitiatedSingleDump) {
  Navigate(shell());

  EXPECT_CALL(*mock_dump_provider_, OnMemoryDump(_,_)).WillOnce(Return(true));

  EnableMemoryTracing();
  RequestGlobalDumpAndWait(true /* from_renderer_thread */);
  EXPECT_EQ(1u, callback_call_count_);
  EXPECT_NE(0u, last_callback_dump_guid_);
  EXPECT_TRUE(last_callback_success_);
  DisableTracing();
}

// This test is flaky on Mac. See crbug.com/522009.
#if defined(OS_MACOSX)
#define MAYBE_ManyInterleavedDumps DISABLED_ManyInterleavedDumps
#else
#define MAYBE_ManyInterleavedDumps ManyInterleavedDumps
#endif
IN_PROC_BROWSER_TEST_F(SingleProcessMemoryTracingTest,
                       MAYBE_ManyInterleavedDumps) {
  Navigate(shell());

  EXPECT_CALL(*mock_dump_provider_, OnMemoryDump(_,_))
      .Times(4)
      .WillRepeatedly(Return(true));

  EnableMemoryTracing();

  RequestGlobalDumpAndWait(true /* from_renderer_thread */);
  EXPECT_NE(0u, last_callback_dump_guid_);
  EXPECT_TRUE(last_callback_success_);

  RequestGlobalDumpAndWait(false /* from_renderer_thread */);
  EXPECT_NE(0u, last_callback_dump_guid_);
  EXPECT_TRUE(last_callback_success_);

  RequestGlobalDumpAndWait(false /* from_renderer_thread */);
  EXPECT_NE(0u, last_callback_dump_guid_);
  EXPECT_TRUE(last_callback_success_);

  RequestGlobalDumpAndWait(true /* from_renderer_thread */);
  EXPECT_EQ(4u, callback_call_count_);
  EXPECT_NE(0u, last_callback_dump_guid_);
  EXPECT_TRUE(last_callback_success_);

  DisableTracing();
}

// Checks that a memory dump initiated from a the main browser thread ends up in
// a successful dump.
IN_PROC_BROWSER_TEST_F(MemoryTracingTest, BrowserInitiatedDump) {
  Navigate(shell());

  EXPECT_CALL(*mock_dump_provider_, OnMemoryDump(_,_)).WillOnce(Return(true));

  EnableMemoryTracing();
  RequestGlobalDumpAndWait(false /* from_renderer_thread */);
  EXPECT_EQ(1u, callback_call_count_);
  EXPECT_NE(0u, last_callback_dump_guid_);
  EXPECT_TRUE(last_callback_success_);
  DisableTracing();
}

}  // namespace content
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC.
// Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.

#include "src/pdf/SkPDFSubsetFont.h"

#if defined(SK_USING_THIRD_PARTY_ICU)
#include "SkLoadICU.h"
#endif

#if defined(SK_PDF_USE_HARFBUZZ_SUBSET)

#include "include/private/SkTemplates.h"
#include "include/private/SkTo.h"
#include "src/utils/SkCallableTraits.h"

#include "hb.h"
#include "hb-subset.h"

template <class T, void(*P)(T*)> using resource =
    std::unique_ptr<T, SkFunctionWrapper<skstd::remove_pointer_t<decltype(P)>, P>>;
using HBBlob = resource<hb_blob_t, &hb_blob_destroy>;
using HBFace = resource<hb_face_t, &hb_face_destroy>;
using HBSubsetInput = resource<hb_subset_input_t, &hb_subset_input_destroy>;
using HBSet = resource<hb_set_t, &hb_set_destroy>;

static HBBlob to_blob(sk_sp<SkData> data) {
    using blob_size_t = SkCallableTraits<decltype(hb_blob_create)>::argument<1>::type;
    if (!SkTFitsIn<blob_size_t>(data->size())) {
        return nullptr;
    }
    const char* blobData = static_cast<const char*>(data->data());
    blob_size_t blobSize = SkTo<blob_size_t>(data->size());
    return HBBlob(hb_blob_create(blobData, blobSize,
                                 HB_MEMORY_MODE_READONLY,
                                 data.release(), [](void* p){ ((SkData*)p)->unref(); }));
}

static sk_sp<SkData> to_data(HBBlob blob) {
    if (!blob) {
        return nullptr;
    }
    unsigned int length;
    const char* data = hb_blob_get_data(blob.get(), &length);
    if (!data || !length) {
        return nullptr;
    }
    return SkData::MakeWithProc(data, SkToSizeT(length),
                                [](const void*, void* ctx) { hb_blob_destroy((hb_blob_t*)ctx); },
                                blob.release());
}

static sk_sp<SkData> subset_harfbuzz(sk_sp<SkData> fontData,
                                     const SkPDFGlyphUse& glyphUsage,
                                     int ttcIndex) {
#if defined(SK_USING_THIRD_PARTY_ICU)
    if (!SkLoadICU()) {
        return nullptr;
    }
#endif
    if (!fontData) {
        return nullptr;
    }
    HBFace face(hb_face_create(to_blob(std::move(fontData)).get(), ttcIndex));
    SkASSERT(face);

    HBSubsetInput input(hb_subset_input_create_or_fail());
    SkASSERT(input);
    if (!face || !input) {
        return nullptr;
    }
    hb_set_t* glyphs = hb_subset_input_glyph_set(input.get());
    glyphUsage.getSetValues([&glyphs](unsigned gid) { hb_set_add(glyphs, gid);});

    hb_subset_input_set_retain_gids(input.get(), true);
    hb_subset_input_set_drop_hints(input.get(), true);
    HBFace subset(hb_subset(face.get(), input.get()));
    HBBlob result(hb_face_reference_blob(subset.get()));
    return to_data(std::move(result));
}

#endif  // defined(SK_PDF_USE_HARFBUZZ_SUBSET)

////////////////////////////////////////////////////////////////////////////////

#if defined(SK_PDF_USE_SFNTLY)

#include "sample/chromium/font_subsetter.h"
#include <vector>

static sk_sp<SkData> subset_sfntly(sk_sp<SkData> fontData,
                                   const SkPDFGlyphUse& glyphUsage,
                                   const char* fontName,
                                   int ttcIndex) {
#if defined(SK_USING_THIRD_PARTY_ICU)
    if (!SkLoadICU()) {
        return nullptr;
    }
#endif
    // Generate glyph id array in format needed by sfntly.
    // TODO(halcanary): sfntly should take a more compact format.
    std::vector<unsigned> subset;
    glyphUsage.getSetValues([&subset](unsigned v) { subset.push_back(v); });

    unsigned char* subsetFont{nullptr};
#if defined(SK_BUILD_FOR_GOOGLE3)
    // TODO(halcanary): update SK_BUILD_FOR_GOOGLE3 to newest version of Sfntly.
    (void)ttcIndex;
    int subsetFontSize = SfntlyWrapper::SubsetFont(fontName,
                                                   fontData->bytes(),
                                                   fontData->size(),
                                                   subset.data(),
                                                   subset.size(),
                                                   &subsetFont);
#else  // defined(SK_BUILD_FOR_GOOGLE3)
    (void)fontName;
    int subsetFontSize = SfntlyWrapper::SubsetFont(ttcIndex,
                                                   fontData->bytes(),
                                                   fontData->size(),
                                                   subset.data(),
                                                   subset.size(),
                                                   &subsetFont);
#endif  // defined(SK_BUILD_FOR_GOOGLE3)
    SkASSERT(subsetFontSize > 0 || subsetFont == nullptr);
    if (subsetFontSize < 1 || subsetFont == nullptr) {
        return nullptr;
    }
    return SkData::MakeWithProc(subsetFont, subsetFontSize,
                                [](const void* p, void*) { delete[] (unsigned char*)p; },
                                nullptr);
}

#endif  // defined(SK_PDF_USE_SFNTLY)

////////////////////////////////////////////////////////////////////////////////

#if defined(SK_PDF_USE_SFNTLY) && defined(SK_PDF_USE_HARFBUZZ_SUBSET)

sk_sp<SkData> SkPDFSubsetFont(sk_sp<SkData> fontData,
                              const SkPDFGlyphUse& glyphUsage,
                              SkPDF::Metadata::Subsetter subsetter,
                              const char* fontName,
                              int ttcIndex) {
    switch (subsetter) {
        case SkPDF::Metadata::kHarfbuzz_Subsetter:
            return subset_harfbuzz(std::move(fontData), glyphUsage, ttcIndex);
        case SkPDF::Metadata::kSfntly_Subsetter:
            return subset_sfntly(std::move(fontData), glyphUsage, fontName, ttcIndex);
    }
    return nullptr;
}

#elif defined(SK_PDF_USE_SFNTLY)

sk_sp<SkData> SkPDFSubsetFont(sk_sp<SkData> fontData,
                              const SkPDFGlyphUse& glyphUsage,
                              SkPDF::Metadata::Subsetter,
                              const char* fontName,
                              int ttcIndex) {
    return subset_sfntly(std::move(fontData), glyphUsage, fontName, ttcIndex);
}

#elif defined(SK_PDF_USE_HARFBUZZ_SUBSET)

sk_sp<SkData> SkPDFSubsetFont(sk_sp<SkData> fontData,
                              const SkPDFGlyphUse& glyphUsage,
                              SkPDF::Metadata::Subsetter,
                              const char*,
                              int ttcIndex) {
    return subset_harfbuzz(std::move(fontData), glyphUsage, ttcIndex);
}

#else

sk_sp<SkData> SkPDFSubsetFont(sk_sp<SkData>, const SkPDFGlyphUse&, SkPDF::Metadata::Subsetter,
                              const char*, int) {
    return nullptr;
}
#endif  // defined(SK_PDF_USE_SFNTLY)

<commit_msg>Retain hints when subsetting for PDF.<commit_after>// Copyright 2018 Google LLC.
// Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.

#include "src/pdf/SkPDFSubsetFont.h"

#if defined(SK_USING_THIRD_PARTY_ICU)
#include "SkLoadICU.h"
#endif

#if defined(SK_PDF_USE_HARFBUZZ_SUBSET)

#include "include/private/SkTemplates.h"
#include "include/private/SkTo.h"
#include "src/utils/SkCallableTraits.h"

#include "hb.h"
#include "hb-subset.h"

template <class T, void(*P)(T*)> using resource =
    std::unique_ptr<T, SkFunctionWrapper<skstd::remove_pointer_t<decltype(P)>, P>>;
using HBBlob = resource<hb_blob_t, &hb_blob_destroy>;
using HBFace = resource<hb_face_t, &hb_face_destroy>;
using HBSubsetInput = resource<hb_subset_input_t, &hb_subset_input_destroy>;
using HBSet = resource<hb_set_t, &hb_set_destroy>;

static HBBlob to_blob(sk_sp<SkData> data) {
    using blob_size_t = SkCallableTraits<decltype(hb_blob_create)>::argument<1>::type;
    if (!SkTFitsIn<blob_size_t>(data->size())) {
        return nullptr;
    }
    const char* blobData = static_cast<const char*>(data->data());
    blob_size_t blobSize = SkTo<blob_size_t>(data->size());
    return HBBlob(hb_blob_create(blobData, blobSize,
                                 HB_MEMORY_MODE_READONLY,
                                 data.release(), [](void* p){ ((SkData*)p)->unref(); }));
}

static sk_sp<SkData> to_data(HBBlob blob) {
    if (!blob) {
        return nullptr;
    }
    unsigned int length;
    const char* data = hb_blob_get_data(blob.get(), &length);
    if (!data || !length) {
        return nullptr;
    }
    return SkData::MakeWithProc(data, SkToSizeT(length),
                                [](const void*, void* ctx) { hb_blob_destroy((hb_blob_t*)ctx); },
                                blob.release());
}

static sk_sp<SkData> subset_harfbuzz(sk_sp<SkData> fontData,
                                     const SkPDFGlyphUse& glyphUsage,
                                     int ttcIndex) {
#if defined(SK_USING_THIRD_PARTY_ICU)
    if (!SkLoadICU()) {
        return nullptr;
    }
#endif
    if (!fontData) {
        return nullptr;
    }
    HBFace face(hb_face_create(to_blob(std::move(fontData)).get(), ttcIndex));
    SkASSERT(face);

    HBSubsetInput input(hb_subset_input_create_or_fail());
    SkASSERT(input);
    if (!face || !input) {
        return nullptr;
    }
    hb_set_t* glyphs = hb_subset_input_glyph_set(input.get());
    glyphUsage.getSetValues([&glyphs](unsigned gid) { hb_set_add(glyphs, gid);});

    hb_subset_input_set_retain_gids(input.get(), true);
    // TODO: When possible, check if a font is 'tricky' with FT_IS_TRICKY.
    // If it isn't known if a font is 'tricky', retain the hints.
    hb_subset_input_set_drop_hints(input.get(), false);
    HBFace subset(hb_subset(face.get(), input.get()));
    HBBlob result(hb_face_reference_blob(subset.get()));
    return to_data(std::move(result));
}

#endif  // defined(SK_PDF_USE_HARFBUZZ_SUBSET)

////////////////////////////////////////////////////////////////////////////////

#if defined(SK_PDF_USE_SFNTLY)

#include "sample/chromium/font_subsetter.h"
#include <vector>

static sk_sp<SkData> subset_sfntly(sk_sp<SkData> fontData,
                                   const SkPDFGlyphUse& glyphUsage,
                                   const char* fontName,
                                   int ttcIndex) {
#if defined(SK_USING_THIRD_PARTY_ICU)
    if (!SkLoadICU()) {
        return nullptr;
    }
#endif
    // Generate glyph id array in format needed by sfntly.
    // TODO(halcanary): sfntly should take a more compact format.
    std::vector<unsigned> subset;
    glyphUsage.getSetValues([&subset](unsigned v) { subset.push_back(v); });

    unsigned char* subsetFont{nullptr};
#if defined(SK_BUILD_FOR_GOOGLE3)
    // TODO(halcanary): update SK_BUILD_FOR_GOOGLE3 to newest version of Sfntly.
    (void)ttcIndex;
    int subsetFontSize = SfntlyWrapper::SubsetFont(fontName,
                                                   fontData->bytes(),
                                                   fontData->size(),
                                                   subset.data(),
                                                   subset.size(),
                                                   &subsetFont);
#else  // defined(SK_BUILD_FOR_GOOGLE3)
    (void)fontName;
    int subsetFontSize = SfntlyWrapper::SubsetFont(ttcIndex,
                                                   fontData->bytes(),
                                                   fontData->size(),
                                                   subset.data(),
                                                   subset.size(),
                                                   &subsetFont);
#endif  // defined(SK_BUILD_FOR_GOOGLE3)
    SkASSERT(subsetFontSize > 0 || subsetFont == nullptr);
    if (subsetFontSize < 1 || subsetFont == nullptr) {
        return nullptr;
    }
    return SkData::MakeWithProc(subsetFont, subsetFontSize,
                                [](const void* p, void*) { delete[] (unsigned char*)p; },
                                nullptr);
}

#endif  // defined(SK_PDF_USE_SFNTLY)

////////////////////////////////////////////////////////////////////////////////

#if defined(SK_PDF_USE_SFNTLY) && defined(SK_PDF_USE_HARFBUZZ_SUBSET)

sk_sp<SkData> SkPDFSubsetFont(sk_sp<SkData> fontData,
                              const SkPDFGlyphUse& glyphUsage,
                              SkPDF::Metadata::Subsetter subsetter,
                              const char* fontName,
                              int ttcIndex) {
    switch (subsetter) {
        case SkPDF::Metadata::kHarfbuzz_Subsetter:
            return subset_harfbuzz(std::move(fontData), glyphUsage, ttcIndex);
        case SkPDF::Metadata::kSfntly_Subsetter:
            return subset_sfntly(std::move(fontData), glyphUsage, fontName, ttcIndex);
    }
    return nullptr;
}

#elif defined(SK_PDF_USE_SFNTLY)

sk_sp<SkData> SkPDFSubsetFont(sk_sp<SkData> fontData,
                              const SkPDFGlyphUse& glyphUsage,
                              SkPDF::Metadata::Subsetter,
                              const char* fontName,
                              int ttcIndex) {
    return subset_sfntly(std::move(fontData), glyphUsage, fontName, ttcIndex);
}

#elif defined(SK_PDF_USE_HARFBUZZ_SUBSET)

sk_sp<SkData> SkPDFSubsetFont(sk_sp<SkData> fontData,
                              const SkPDFGlyphUse& glyphUsage,
                              SkPDF::Metadata::Subsetter,
                              const char*,
                              int ttcIndex) {
    return subset_harfbuzz(std::move(fontData), glyphUsage, ttcIndex);
}

#else

sk_sp<SkData> SkPDFSubsetFont(sk_sp<SkData>, const SkPDFGlyphUse&, SkPDF::Metadata::Subsetter,
                              const char*, int) {
    return nullptr;
}
#endif  // defined(SK_PDF_USE_SFNTLY)

<|endoftext|>


<commit_before>#ifdef WIN32
#include <windows.h>
#include <XGetopt.h>
#else
#include <unistd.h>
#endif
#include <stdint.h>
#include <iostream>
#include <vector>
#include <regex>
#include <codecvt>
#include <Directory.h>
#include <Counter.h>
#include <File.h>

int32_t main(int32_t argc, char *argv[ ])
{
    std::wstring regex = L".*\\.(cc|cpp|h)";
    std::wstring dir = L".";
    std::wstring textFileDir = L"";

    int32_t c;

    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;

    while ((c = getopt(argc, argv, "e:d:t:h")) != EOF)
        if(c == 'e')
            regex = converter.from_bytes(optarg);
        else if(c == 'd')
            dir = converter.from_bytes(optarg);
        else if(c == 't')
            textFileDir = converter.from_bytes(optarg);
        else if(c == '?' || c == 'h'){
            std::cout << "usage: countLines [-e regexp] [-d directory] [-t text]" << std::endl;
            std::cout << " -e: file regular expression pattern" << std::endl;
            std::cout << " -d: root processing directory" << std::endl;
            std::cout << " -t: text file dir. This text will be placed at the beginning of each" << std::endl;
            std::cout << "     file that matches regular expression" << std::endl;
        }

    Counter::Counter counter;

    try{

        std::string text;

        if(textFileDir != L""){
            File::Data data = File::Read(textFileDir);
            std::for_each(data.begin(), data.end(), [&text](File::Data::value_type Value){text += Value;});
        }

        counter.Init(regex, dir, text);
        counter.Process();

    }catch(const std::regex_error &err){
        std::cout << "regex syntax error " << err.what();
        return 1;
    }catch(const Exception &err){
        std::wcout << L"processing error " << err.What();
        return 1;
    }

    std::cout << "lines count " << counter.GetLinesCount() << std::endl;
    return 0;
}<commit_msg>close #2<commit_after>#ifdef WIN32
#include <windows.h>
#include <XGetopt.h>
#else
#include <unistd.h>
#endif
#include <stdint.h>
#include <iostream>
#include <vector>
#include <regex>
#include <codecvt>
#include <Directory.h>
#include <Counter.h>
#include <File.h>

static void PrintUsage()
{
    std::cout << "usage: countLines [-e regexp] [-d directory] [-t text]" << std::endl;
    std::cout << " -e: file regular expression pattern" << std::endl;
    std::cout << " -d: root processing directory" << std::endl;
    std::cout << " -t: text file dir. This text will be placed at the beginning of each" << std::endl;
    std::cout << "     file that matches regular expression" << std::endl;

    exit(0);
}

int32_t main(int32_t argc, char *argv[ ])
{
    std::wstring regex = L".*\\.(cc|cpp|h)";
    std::wstring dir = L".";
    std::wstring textFileDir = L"";

    int32_t c;

    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;

    while ((c = getopt(argc, argv, "e:d:t:h")) != EOF)
        if(c == 'e')
            regex = converter.from_bytes(optarg);
        else if(c == 'd')
            dir = converter.from_bytes(optarg);
        else if(c == 't')
            textFileDir = converter.from_bytes(optarg);
        else if(c == '?' || c == 'h')
            PrintUsage();
        else {
            std::cout << "invalid option " << c << std::endl;
            PrintUsage();
        }

    Counter::Counter counter;

    try{

        std::string text;

        if(textFileDir != L""){
            File::Data data = File::Read(textFileDir);
            std::for_each(data.begin(), data.end(), [&text](File::Data::value_type Value){text += Value;});
        }

        counter.Init(regex, dir, text);
        counter.Process();

    }catch(const std::regex_error &err){
        std::cout << "regex syntax error " << err.what();
        return 1;
    }catch(const Exception &err){
        std::wcout << L"processing error " << err.What();
        return 1;
    }

    std::cout << "lines count " << counter.GetLinesCount() << std::endl;
    return 0;
}<|endoftext|>


<commit_before>// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "content/shell/renderer/test_runner/MockSpellCheck.h"

#include "content/shell/renderer/test_runner/TestCommon.h"
#include "third_party/WebKit/public/platform/WebCString.h"

using namespace blink;
using namespace std;

namespace WebTestRunner {

namespace {

void append(WebVector<WebString>* data, const WebString& item)
{
    WebVector<WebString> result(data->size() + 1);
    for (size_t i = 0; i < data->size(); ++i)
        result[i] = (*data)[i];
    result[data->size()] = item;
    data->swap(result);
}

}

MockSpellCheck::MockSpellCheck()
    : m_initialized(false) { }

MockSpellCheck::~MockSpellCheck() { }

bool MockSpellCheck::spellCheckWord(const WebString& text, int* misspelledOffset, int* misspelledLength)
{
    BLINK_ASSERT(misspelledOffset);
    BLINK_ASSERT(misspelledLength);

    // Initialize this spellchecker.
    initializeIfNeeded();

    // Reset the result values as our spellchecker does.
    *misspelledOffset = 0;
    *misspelledLength = 0;

    // Convert to a base::string16 because we store base::string16 instances in
    // m_misspelledWords and WebString has no find().
    base::string16 stringText = text;
    int skippedLength = 0;

    while (!stringText.empty()) {
        // Extract the first possible English word from the given string.
        // The given string may include non-ASCII characters or numbers. So, we
        // should filter out such characters before start looking up our
        // misspelled-word table.
        // (This is a simple version of our SpellCheckWordIterator class.)
        // If the given string doesn't include any ASCII characters, we can treat the
        // string as valid one.
        base::string16::iterator firstChar = find_if(stringText.begin(), stringText.end(), isASCIIAlpha);
        if (firstChar == stringText.end())
            return true;
        int wordOffset = distance(stringText.begin(), firstChar);
        int maxWordLength = static_cast<int>(stringText.length()) - wordOffset;
        int wordLength;
        base::string16 word;

        // Look up our misspelled-word table to check if the extracted word is a
        // known misspelled word, and return the offset and the length of the
        // extracted word if this word is a known misspelled word.
        // (See the comment in MockSpellCheck::initializeIfNeeded() why we use a
        // misspelled-word table.)
        for (size_t i = 0; i < m_misspelledWords.size(); ++i) {
            wordLength = static_cast<int>(m_misspelledWords.at(i).length()) > maxWordLength ? maxWordLength : static_cast<int>(m_misspelledWords.at(i).length());
            word = stringText.substr(wordOffset, wordLength);
            if (word == m_misspelledWords.at(i) && (static_cast<int>(stringText.length()) == wordOffset + wordLength || isNotASCIIAlpha(stringText[wordOffset + wordLength]))) {
                *misspelledOffset = wordOffset + skippedLength;
                *misspelledLength = wordLength;
                break;
            }
        }

        if (*misspelledLength > 0)
            break;

        base::string16::iterator lastChar = find_if(stringText.begin() + wordOffset, stringText.end(), isNotASCIIAlpha);
        if (lastChar == stringText.end())
            wordLength = static_cast<int>(stringText.length()) - wordOffset;
        else
            wordLength = distance(firstChar, lastChar);

        BLINK_ASSERT(0 < wordOffset + wordLength);
        stringText = stringText.substr(wordOffset + wordLength);
        skippedLength += wordOffset + wordLength;
    }

    return false;
}

bool MockSpellCheck::hasInCache(const WebString& word)
{
    return word == WebString::fromUTF8("Spell wellcome. Is it broken?") || word == WebString::fromUTF8("Spell wellcome.\x007F");
}

bool MockSpellCheck::isMultiWordMisspelling(const WebString& text, vector<WebTextCheckingResult>* results)
{
    if (text == WebString::fromUTF8("Helllo wordl.")) {
        results->push_back(WebTextCheckingResult(WebTextDecorationTypeSpelling, 0, 6, WebString("Hello")));
        results->push_back(WebTextCheckingResult(WebTextDecorationTypeSpelling, 7, 5, WebString("world")));
        return true;
    }
    return false;
}

void MockSpellCheck::fillSuggestionList(const WebString& word, WebVector<WebString>* suggestions)
{
    if (word == WebString::fromUTF8("wellcome"))
        append(suggestions, WebString::fromUTF8("welcome"));
    else if (word == WebString::fromUTF8("upper case"))
        append(suggestions, WebString::fromUTF8("uppercase"));
    else if (word == WebString::fromUTF8("Helllo"))
        append(suggestions, WebString::fromUTF8("Hello"));
    else if (word == WebString::fromUTF8("wordl"))
        append(suggestions, WebString::fromUTF8("world"));
}

bool MockSpellCheck::initializeIfNeeded()
{
    // Exit if we have already initialized this object.
    if (m_initialized)
        return false;

    // Create a table that consists of misspelled words used in WebKit layout
    // tests.
    // Since WebKit layout tests don't have so many misspelled words as
    // well-spelled words, it is easier to compare the given word with misspelled
    // ones than to compare with well-spelled ones.
    static const char* misspelledWords[] = {
        // These words are known misspelled words in webkit tests.
        // If there are other misspelled words in webkit tests, please add them in
        // this array.
        "foo",
        "Foo",
        "baz",
        "fo",
        "LibertyF",
        "chello",
        "xxxtestxxx",
        "XXxxx",
        "Textx",
        "blockquoted",
        "asd",
        "Lorem",
        "Nunc",
        "Curabitur",
        "eu",
        "adlj",
        "adaasj",
        "sdklj",
        "jlkds",
        "jsaada",
        "jlda",
        "zz",
        "contentEditable",
        // The following words are used by unit tests.
        "ifmmp",
        "qwertyuiopasd",
        "qwertyuiopasdf",
        "upper case",
        "wellcome"
    };

    m_misspelledWords.clear();
    for (size_t i = 0; i < arraysize(misspelledWords); ++i)
        m_misspelledWords.push_back(base::string16(misspelledWords[i], misspelledWords[i] + strlen(misspelledWords[i])));

    // Mark as initialized to prevent this object from being initialized twice
    // or more.
    m_initialized = true;

    // Since this MockSpellCheck class doesn't download dictionaries, this
    // function always returns false.
    return false;
}

}
<commit_msg>Fix typo in MockSpellCheck::misspelledWords<commit_after>// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "content/shell/renderer/test_runner/MockSpellCheck.h"

#include "content/shell/renderer/test_runner/TestCommon.h"
#include "third_party/WebKit/public/platform/WebCString.h"

using namespace blink;
using namespace std;

namespace WebTestRunner {

namespace {

void append(WebVector<WebString>* data, const WebString& item)
{
    WebVector<WebString> result(data->size() + 1);
    for (size_t i = 0; i < data->size(); ++i)
        result[i] = (*data)[i];
    result[data->size()] = item;
    data->swap(result);
}

}

MockSpellCheck::MockSpellCheck()
    : m_initialized(false) { }

MockSpellCheck::~MockSpellCheck() { }

bool MockSpellCheck::spellCheckWord(const WebString& text, int* misspelledOffset, int* misspelledLength)
{
    BLINK_ASSERT(misspelledOffset);
    BLINK_ASSERT(misspelledLength);

    // Initialize this spellchecker.
    initializeIfNeeded();

    // Reset the result values as our spellchecker does.
    *misspelledOffset = 0;
    *misspelledLength = 0;

    // Convert to a base::string16 because we store base::string16 instances in
    // m_misspelledWords and WebString has no find().
    base::string16 stringText = text;
    int skippedLength = 0;

    while (!stringText.empty()) {
        // Extract the first possible English word from the given string.
        // The given string may include non-ASCII characters or numbers. So, we
        // should filter out such characters before start looking up our
        // misspelled-word table.
        // (This is a simple version of our SpellCheckWordIterator class.)
        // If the given string doesn't include any ASCII characters, we can treat the
        // string as valid one.
        base::string16::iterator firstChar = find_if(stringText.begin(), stringText.end(), isASCIIAlpha);
        if (firstChar == stringText.end())
            return true;
        int wordOffset = distance(stringText.begin(), firstChar);
        int maxWordLength = static_cast<int>(stringText.length()) - wordOffset;
        int wordLength;
        base::string16 word;

        // Look up our misspelled-word table to check if the extracted word is a
        // known misspelled word, and return the offset and the length of the
        // extracted word if this word is a known misspelled word.
        // (See the comment in MockSpellCheck::initializeIfNeeded() why we use a
        // misspelled-word table.)
        for (size_t i = 0; i < m_misspelledWords.size(); ++i) {
            wordLength = static_cast<int>(m_misspelledWords.at(i).length()) > maxWordLength ? maxWordLength : static_cast<int>(m_misspelledWords.at(i).length());
            word = stringText.substr(wordOffset, wordLength);
            if (word == m_misspelledWords.at(i) && (static_cast<int>(stringText.length()) == wordOffset + wordLength || isNotASCIIAlpha(stringText[wordOffset + wordLength]))) {
                *misspelledOffset = wordOffset + skippedLength;
                *misspelledLength = wordLength;
                break;
            }
        }

        if (*misspelledLength > 0)
            break;

        base::string16::iterator lastChar = find_if(stringText.begin() + wordOffset, stringText.end(), isNotASCIIAlpha);
        if (lastChar == stringText.end())
            wordLength = static_cast<int>(stringText.length()) - wordOffset;
        else
            wordLength = distance(firstChar, lastChar);

        BLINK_ASSERT(0 < wordOffset + wordLength);
        stringText = stringText.substr(wordOffset + wordLength);
        skippedLength += wordOffset + wordLength;
    }

    return false;
}

bool MockSpellCheck::hasInCache(const WebString& word)
{
    return word == WebString::fromUTF8("Spell wellcome. Is it broken?") || word == WebString::fromUTF8("Spell wellcome.\x007F");
}

bool MockSpellCheck::isMultiWordMisspelling(const WebString& text, vector<WebTextCheckingResult>* results)
{
    if (text == WebString::fromUTF8("Helllo wordl.")) {
        results->push_back(WebTextCheckingResult(WebTextDecorationTypeSpelling, 0, 6, WebString("Hello")));
        results->push_back(WebTextCheckingResult(WebTextDecorationTypeSpelling, 7, 5, WebString("world")));
        return true;
    }
    return false;
}

void MockSpellCheck::fillSuggestionList(const WebString& word, WebVector<WebString>* suggestions)
{
    if (word == WebString::fromUTF8("wellcome"))
        append(suggestions, WebString::fromUTF8("welcome"));
    else if (word == WebString::fromUTF8("upper case"))
        append(suggestions, WebString::fromUTF8("uppercase"));
    else if (word == WebString::fromUTF8("Helllo"))
        append(suggestions, WebString::fromUTF8("Hello"));
    else if (word == WebString::fromUTF8("wordl"))
        append(suggestions, WebString::fromUTF8("world"));
}

bool MockSpellCheck::initializeIfNeeded()
{
    // Exit if we have already initialized this object.
    if (m_initialized)
        return false;

    // Create a table that consists of misspelled words used in WebKit layout
    // tests.
    // Since WebKit layout tests don't have so many misspelled words as
    // well-spelled words, it is easier to compare the given word with misspelled
    // ones than to compare with well-spelled ones.
    static const char* misspelledWords[] = {
        // These words are known misspelled words in webkit tests.
        // If there are other misspelled words in webkit tests, please add them in
        // this array.
        "foo",
        "Foo",
        "baz",
        "fo",
        "LibertyF",
        "chello",
        "xxxtestxxx",
        "XXxxx",
        "Textx",
        "blockquoted",
        "asdf",
        "Lorem",
        "Nunc",
        "Curabitur",
        "eu",
        "adlj",
        "adaasj",
        "sdklj",
        "jlkds",
        "jsaada",
        "jlda",
        "zz",
        "contentEditable",
        // The following words are used by unit tests.
        "ifmmp",
        "qwertyuiopasd",
        "qwertyuiopasdf",
        "upper case",
        "wellcome"
    };

    m_misspelledWords.clear();
    for (size_t i = 0; i < arraysize(misspelledWords); ++i)
        m_misspelledWords.push_back(base::string16(misspelledWords[i], misspelledWords[i] + strlen(misspelledWords[i])));

    // Mark as initialized to prevent this object from being initialized twice
    // or more.
    m_initialized = true;

    // Since this MockSpellCheck class doesn't download dictionaries, this
    // function always returns false.
    return false;
}

}
<|endoftext|>


<commit_before>// Plugging custom error codes into std::error_code etc. machinery example

#include "custom_error_code_bits/the_game.h"
#include <iostream>
#include <string>

void log_bad_status_codes( std::error_code ec )
{
  if ( ec )
  {
    std::clog << ec << " " << ec.message() << "\n";
  }
}

int main()
{
  auto & engine{ the_game::get_appengine() };

  // Should fail as setting renderer supporting invalid dimension range
  std::unique_ptr<the_game::renderer> rend{new the_game::oops_renderer};
  log_bad_status_codes( engine.take_renderer( std::move(rend) ) );

  // Should fail as no renderer successfully set to draw board
  log_bad_status_codes( engine.update_game_board() );

  // OK - nothing to report, this renderer is fine and dandy
  rend.reset( new the_game::fine_renderer );
  log_bad_status_codes( engine.take_renderer(std::move(rend)) );

  // OK - now have renderer to render board updates
  log_bad_status_codes( engine.update_game_board() );
}
<commit_msg>Whitspace change<commit_after>// Plugging custom error codes into std::error_code etc. machinery example

#include "custom_error_code_bits/the_game.h"
#include <system_error>
#include <iostream>
#include <string>

void log_bad_status_codes( std::error_code ec )
{
  if ( ec )
  {
    std::clog << ec << " " << ec.message() << "\n";
  }
}

int main()
{
  auto & engine{ the_game::get_appengine() };

  // Should fail as setting renderer supporting invalid dimension range
  std::unique_ptr<the_game::renderer> rend{new the_game::oops_renderer};
  log_bad_status_codes( engine.take_renderer( std::move(rend) ) );

  // Should fail as no renderer successfully set to draw board
  log_bad_status_codes( engine.update_game_board() );

  // OK - nothing to report, this renderer is fine and dandy
  rend.reset( new the_game::fine_renderer );
  log_bad_status_codes( engine.take_renderer( std::move(rend)) );

  // OK - now have renderer to render board updates
  log_bad_status_codes( engine.update_game_board() );
}
<|endoftext|>


<commit_before><commit_msg>Add a defined(OS_LINUX) guard for the include of tcmalloc's profiler.h<commit_after><|endoftext|>


<commit_before><commit_msg>fix core from a federated query and null msg20<commit_after><|endoftext|>