<commit_before>/**
 * Copyright 2017 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "drive.hpp"

#include "interfaces.hpp"
#include "sensors/pluggable.hpp"
#include "sysfs/sysfsread.hpp"
#include "sysfs/sysfswrite.hpp"

#include <iostream>
#include <memory>
#include <tuple>

using tstamp = std::chrono::high_resolution_clock::time_point;

#define DRIVE_TIME 1
#define DRIVE_GOAL 2
#define DRIVE DRIVE_TIME
#define MAX_PWM 255

static std::unique_ptr<Sensor> Create(std::string readpath,
                                      std::string writepath)
{
    return std::make_unique<PluggableSensor>(
        readpath, 0, /* default the timeout to disabled */
        std::make_unique<SysFsRead>(readpath),
        std::make_unique<SysFsWrite>(writepath, 0, MAX_PWM));
}

int64_t getAverage(std::tuple<tstamp, int64_t, int64_t>& values)
{
    return (std::get<1>(values) + std::get<2>(values)) / 2;
}

bool valueClose(int64_t value, int64_t goal)
{
#if 0
    int64_t delta = 100; /* within 100 */
    if (value < (goal + delta) &&
        value > (goal - delta))
    {
        return true;
    }
#endif

    /* let's make sure it's below goal. */
    if (value < goal)
    {
        return true;
    }

    return false;
}

static void driveGoal(int64_t& seriesCnt, int64_t setPwm, int64_t goal,
                      std::vector<std::tuple<tstamp, int64_t, int64_t>>& series,
                      std::vector<std::unique_ptr<Sensor>>& fanSensors)
{
    bool reading = true;

    auto& fan0 = fanSensors.at(0);
    auto& fan1 = fanSensors.at(1);

    fan0->write(setPwm);
    fan1->write(setPwm);

    while (reading)
    {
        bool check = false;
        ReadReturn r0 = fan0->read();
        ReadReturn r1 = fan1->read();
        int64_t n0 = static_cast<int64_t>(r0.value);
        int64_t n1 = static_cast<int64_t>(r1.value);

        tstamp t1 = std::chrono::high_resolution_clock::now();

        series.push_back(std::make_tuple(t1, n0, n1));
        seriesCnt += 1;

        int64_t avgn = (n0 + n1) / 2;
        /* check last three values against goal if this is close */
        check = valueClose(avgn, goal);

        /* We know the last entry is within range. */
        if (check && seriesCnt > 3)
        {
            /* n-2 values */
            std::tuple<tstamp, int64_t, int64_t> nm2 = series.at(seriesCnt - 3);
            /* n-1 values */
            std::tuple<tstamp, int64_t, int64_t> nm1 = series.at(seriesCnt - 2);

            int64_t avgnm2 = getAverage(nm2);
            int64_t avgnm1 = getAverage(nm1);

            int64_t together = (avgnm2 + avgnm1) / 2;

            reading = !valueClose(together, goal);

            if (!reading)
            {
                std::cerr << "finished reaching goal\n";
            }
        }

        /* Early abort for testing. */
        if (seriesCnt > 150000)
        {
            std::cerr << "aborting after 150000 reads.\n";
            reading = false;
        }
    }

    return;
}

static void driveTime(int64_t& seriesCnt, int64_t setPwm, int64_t goal,
                      std::vector<std::tuple<tstamp, int64_t, int64_t>>& series,
                      std::vector<std::unique_ptr<Sensor>>& fanSensors)
{
    using namespace std::literals::chrono_literals;

    bool reading = true;

    auto& fan0 = fanSensors.at(0);
    auto& fan1 = fanSensors.at(1);

    auto& s0 = series.at(0);
    tstamp t0 = std::get<0>(s0);

    fan0->write(setPwm);
    fan1->write(setPwm);

    while (reading)
    {
        ReadReturn r0 = fan0->read();
        ReadReturn r1 = fan1->read();
        int64_t n0 = static_cast<int64_t>(r0.value);
        int64_t n1 = static_cast<int64_t>(r1.value);
        tstamp t1 = std::chrono::high_resolution_clock::now();

        series.push_back(std::make_tuple(t1, n0, n1));

        auto duration =
            std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0)
                .count();
        if (duration >= (20000000us).count())
        {
            reading = false;
        }
    }

    return;
}

int driveMain(void)
{
    /* Time series of the data, the timestamp after both are read and the
     * values. */
    std::vector<std::tuple<tstamp, int64_t, int64_t>> series;
    int64_t seriesCnt = 0; /* in case vector count isn't constant time */
    int drive = DRIVE;

    /*
     * The fan map:
     *  --> 0 | 4
     *  --> 1 | 5
     *  --> 2 | 6
     *  --> 3 | 7
     */
    std::vector<std::string> fans = {"/sys/class/hwmon/hwmon0/fan0_input",
                                     "/sys/class/hwmon/hwmon0/fan4_input"};

    std::vector<std::string> pwms = {"/sys/class/hwmon/hwmon0/pwm0",
                                     "/sys/class/hwmon/hwmon0/pwm4"};

    std::vector<std::unique_ptr<Sensor>> fanSensors;

    auto fan0 = Create(fans[0], pwms[0]);
    auto fan1 = Create(fans[1], pwms[1]);

    ReadReturn r0 = fan0->read();
    ReadReturn r1 = fan1->read();
    int64_t pwm0_value = static_cast<int64_t>(r0.value);
    int64_t pwm1_value = static_cast<int64_t>(r1.value);

    if (MAX_PWM != pwm0_value || MAX_PWM != pwm1_value)
    {
        std::cerr << "bad PWM starting point.\n";
        return -EINVAL;
    }

    r0 = fan0->read();
    r1 = fan1->read();
    int64_t fan0_start = r0.value;
    int64_t fan1_start = r1.value;
    tstamp t1 = std::chrono::high_resolution_clock::now();

    /*
     * I've done experiments, and seen 9080,10243 as a starting point
     * which leads to a 50% goal of 4830.5, which is higher than the
     * average that they reach, 4668.  -- i guess i could try to figure out
     * a good increase from one to the other, but how fast they're going
     * actually influences how much they influence, so at slower speeds the
     * improvement is less.
     */

    series.push_back(std::make_tuple(t1, fan0_start, fan1_start));
    seriesCnt += 1;

    int64_t average = (fan0_start + fan1_start) / 2;
    int64_t goal = 0.5 * average;

    std::cerr << "goal: " << goal << "\n";

    // fan0 @ 128: 4691
    // fan4 @ 128: 4707

    fanSensors.push_back(std::move(fan0));
    fanSensors.push_back(std::move(fan1));

    if (DRIVE_TIME == drive)
    {
        driveTime(seriesCnt, 128, goal, series, fanSensors);
    }
    else if (DRIVE_GOAL == drive)
    {
        driveGoal(seriesCnt, 128, goal, series, fanSensors);
    }
    tstamp tp = t1;

    /* Output the values and the timepoints as a time series for review. */
    for (const auto& t : series)
    {
        tstamp ts = std::get<0>(t);
        int64_t n0 = std::get<1>(t);
        int64_t n1 = std::get<2>(t);

        auto duration =
            std::chrono::duration_cast<std::chrono::microseconds>(ts - tp)
                .count();
        std::cout << duration << "us, " << n0 << ", " << n1 << "\n";

        tp = ts;
    }

    return 0;
}
<commit_msg>experiment: cppcheck cleanup<commit_after>/**
 * Copyright 2017 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "drive.hpp"

#include "interfaces.hpp"
#include "sensors/pluggable.hpp"
#include "sysfs/sysfsread.hpp"
#include "sysfs/sysfswrite.hpp"

#include <iostream>
#include <memory>
#include <tuple>

using tstamp = std::chrono::high_resolution_clock::time_point;

#define DRIVE_TIME 1
#define DRIVE_GOAL 2
#define DRIVE DRIVE_TIME
#define MAX_PWM 255

static std::unique_ptr<Sensor> Create(std::string readpath,
                                      std::string writepath)
{
    return std::make_unique<PluggableSensor>(
        readpath, 0, /* default the timeout to disabled */
        std::make_unique<SysFsRead>(readpath),
        std::make_unique<SysFsWrite>(writepath, 0, MAX_PWM));
}

int64_t getAverage(std::tuple<tstamp, int64_t, int64_t>& values)
{
    return (std::get<1>(values) + std::get<2>(values)) / 2;
}

bool valueClose(int64_t value, int64_t goal)
{
#if 0
    int64_t delta = 100; /* within 100 */
    if (value < (goal + delta) &&
        value > (goal - delta))
    {
        return true;
    }
#endif

    /* let's make sure it's below goal. */
    if (value < goal)
    {
        return true;
    }

    return false;
}

static void driveGoal(int64_t& seriesCnt, int64_t setPwm, int64_t goal,
                      std::vector<std::tuple<tstamp, int64_t, int64_t>>& series,
                      std::vector<std::unique_ptr<Sensor>>& fanSensors)
{
    bool reading = true;

    auto& fan0 = fanSensors.at(0);
    auto& fan1 = fanSensors.at(1);

    fan0->write(setPwm);
    fan1->write(setPwm);

    while (reading)
    {
        bool check;
        ReadReturn r0 = fan0->read();
        ReadReturn r1 = fan1->read();
        int64_t n0 = static_cast<int64_t>(r0.value);
        int64_t n1 = static_cast<int64_t>(r1.value);

        tstamp t1 = std::chrono::high_resolution_clock::now();

        series.push_back(std::make_tuple(t1, n0, n1));
        seriesCnt += 1;

        int64_t avgn = (n0 + n1) / 2;
        /* check last three values against goal if this is close */
        check = valueClose(avgn, goal);

        /* We know the last entry is within range. */
        if (check && seriesCnt > 3)
        {
            /* n-2 values */
            std::tuple<tstamp, int64_t, int64_t> nm2 = series.at(seriesCnt - 3);
            /* n-1 values */
            std::tuple<tstamp, int64_t, int64_t> nm1 = series.at(seriesCnt - 2);

            int64_t avgnm2 = getAverage(nm2);
            int64_t avgnm1 = getAverage(nm1);

            int64_t together = (avgnm2 + avgnm1) / 2;

            reading = !valueClose(together, goal);

            if (!reading)
            {
                std::cerr << "finished reaching goal\n";
            }
        }

        /* Early abort for testing. */
        if (seriesCnt > 150000)
        {
            std::cerr << "aborting after 150000 reads.\n";
            reading = false;
        }
    }

    return;
}

static void driveTime(int64_t& seriesCnt, int64_t setPwm, int64_t goal,
                      std::vector<std::tuple<tstamp, int64_t, int64_t>>& series,
                      std::vector<std::unique_ptr<Sensor>>& fanSensors)
{
    using namespace std::literals::chrono_literals;

    bool reading = true;

    auto& fan0 = fanSensors.at(0);
    auto& fan1 = fanSensors.at(1);

    auto& s0 = series.at(0);
    tstamp t0 = std::get<0>(s0);

    fan0->write(setPwm);
    fan1->write(setPwm);

    while (reading)
    {
        ReadReturn r0 = fan0->read();
        ReadReturn r1 = fan1->read();
        int64_t n0 = static_cast<int64_t>(r0.value);
        int64_t n1 = static_cast<int64_t>(r1.value);
        tstamp t1 = std::chrono::high_resolution_clock::now();

        series.push_back(std::make_tuple(t1, n0, n1));

        auto duration =
            std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0)
                .count();
        if (duration >= (20000000us).count())
        {
            reading = false;
        }
    }

    return;
}

int driveMain(void)
{
    /* Time series of the data, the timestamp after both are read and the
     * values. */
    std::vector<std::tuple<tstamp, int64_t, int64_t>> series;
    int64_t seriesCnt = 0; /* in case vector count isn't constant time */
    int drive = DRIVE;

    /*
     * The fan map:
     *  --> 0 | 4
     *  --> 1 | 5
     *  --> 2 | 6
     *  --> 3 | 7
     */
    std::vector<std::string> fans = {"/sys/class/hwmon/hwmon0/fan0_input",
                                     "/sys/class/hwmon/hwmon0/fan4_input"};

    std::vector<std::string> pwms = {"/sys/class/hwmon/hwmon0/pwm0",
                                     "/sys/class/hwmon/hwmon0/pwm4"};

    std::vector<std::unique_ptr<Sensor>> fanSensors;

    auto fan0 = Create(fans[0], pwms[0]);
    auto fan1 = Create(fans[1], pwms[1]);

    ReadReturn r0 = fan0->read();
    ReadReturn r1 = fan1->read();
    int64_t pwm0_value = static_cast<int64_t>(r0.value);
    int64_t pwm1_value = static_cast<int64_t>(r1.value);

    if (MAX_PWM != pwm0_value || MAX_PWM != pwm1_value)
    {
        std::cerr << "bad PWM starting point.\n";
        return -EINVAL;
    }

    r0 = fan0->read();
    r1 = fan1->read();
    int64_t fan0_start = r0.value;
    int64_t fan1_start = r1.value;
    tstamp t1 = std::chrono::high_resolution_clock::now();

    /*
     * I've done experiments, and seen 9080,10243 as a starting point
     * which leads to a 50% goal of 4830.5, which is higher than the
     * average that they reach, 4668.  -- i guess i could try to figure out
     * a good increase from one to the other, but how fast they're going
     * actually influences how much they influence, so at slower speeds the
     * improvement is less.
     */

    series.push_back(std::make_tuple(t1, fan0_start, fan1_start));
    seriesCnt += 1;

    int64_t average = (fan0_start + fan1_start) / 2;
    int64_t goal = 0.5 * average;

    std::cerr << "goal: " << goal << "\n";

    // fan0 @ 128: 4691
    // fan4 @ 128: 4707

    fanSensors.push_back(std::move(fan0));
    fanSensors.push_back(std::move(fan1));

    if (DRIVE_TIME == drive)
    {
        driveTime(seriesCnt, 128, goal, series, fanSensors);
    }
    else if (DRIVE_GOAL == drive)
    {
        driveGoal(seriesCnt, 128, goal, series, fanSensors);
    }
    tstamp tp = t1;

    /* Output the values and the timepoints as a time series for review. */
    for (const auto& t : series)
    {
        tstamp ts = std::get<0>(t);
        int64_t n0 = std::get<1>(t);
        int64_t n1 = std::get<2>(t);

        auto duration =
            std::chrono::duration_cast<std::chrono::microseconds>(ts - tp)
                .count();
        std::cout << duration << "us, " << n0 << ", " << n1 << "\n";

        tp = ts;
    }

    return 0;
}
<|endoftext|>


<commit_before>// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "util/webserver.h"

#include <boost/algorithm/string.hpp>
#include <boost/bind.hpp>
#include <boost/filesystem.hpp>
#include <boost/foreach.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/mem_fn.hpp>
#include <boost/thread/locks.hpp>
#include <gutil/strings/substitute.h>
#include <map>
#include <fstream>
#include <stdio.h>
#include <signal.h>
#include <string>
#include <mustache/mustache.h>
#include <rapidjson/stringbuffer.h>
#include <rapidjson/prettywriter.h>

#include "common/logging.h"
#include "util/cpu-info.h"
#include "util/disk-info.h"
#include "util/mem-info.h"
#include "util/os-info.h"
#include "util/url-coding.h"
#include "util/debug-util.h"
#include "util/stopwatch.h"
#include "rpc/thrift-util.h"

using namespace std;
using namespace boost;
using namespace boost::filesystem;
using namespace google;
using namespace strings;
using namespace rapidjson;
using namespace mustache;

const char* GetDefaultDocumentRoot();

DEFINE_int32(webserver_port, 25000, "Port to start debug webserver on");
DEFINE_string(webserver_interface, "",
    "Interface to start debug webserver on. If blank, webserver binds to 0.0.0.0");
DEFINE_string(webserver_doc_root, GetDefaultDocumentRoot(),
    "Files under <webserver_doc_root>/www are accessible via the debug webserver. "
    "Defaults to $IMPALA_HOME, or if $IMPALA_HOME is not set, disables the document "
    "root");
DEFINE_bool(enable_webserver_doc_root, true,
    "If true, webserver may serve static files from the webserver_doc_root");

DEFINE_string(webserver_certificate_file, "",
    "The location of the debug webserver's SSL certificate file, in .pem format. If "
    "empty, webserver SSL support is not enabled");
DEFINE_string(webserver_authentication_domain, "",
    "Domain used for debug webserver authentication");
DEFINE_string(webserver_password_file, "",
    "(Optional) Location of .htpasswd file containing user names and hashed passwords for"
    " debug webserver authentication");

static const char* DOC_FOLDER = "/www/";
static const int DOC_FOLDER_LEN = strlen(DOC_FOLDER);

// Easy-to-read constants for Squeasel return codes
static const uint32_t PROCESSING_COMPLETE = 1;
static const uint32_t NOT_PROCESSED = 0;

// Standard keys in the json document sent to templates for rendering. Must be kept in
// sync with the templates themselves.
static const char* COMMON_JSON_KEY = "__common__";

// Returns $IMPALA_HOME if set, otherwise /tmp/impala_www
const char* GetDefaultDocumentRoot() {
  stringstream ss;
  char* impala_home = getenv("IMPALA_HOME");
  if (impala_home == NULL) {
    return ""; // Empty document root means don't serve static files
  } else {
    ss << impala_home;
  }

  // Deliberate memory leak, but this should be called exactly once.
  string* str = new string(ss.str());
  return str->c_str();
}

namespace impala {

const char* Webserver::ENABLE_RAW_JSON_KEY = "__raw__";

Webserver::Webserver() : context_(NULL) {
  http_address_ = MakeNetworkAddress(
      FLAGS_webserver_interface.empty() ? "0.0.0.0" : FLAGS_webserver_interface,
      FLAGS_webserver_port);
}

Webserver::Webserver(const int port) : context_(NULL) {
  http_address_ = MakeNetworkAddress("0.0.0.0", port);
}

Webserver::~Webserver() {
  Stop();
}

void Webserver::RootHandler(const Webserver::ArgumentMap& args, Document* document) {
  document->AddMember("version", GetVersionString().c_str(),
      document->GetAllocator());
  document->AddMember("cpu_info", CpuInfo::DebugString().c_str(),
      document->GetAllocator());
  document->AddMember("mem_info", MemInfo::DebugString().c_str(),
      document->GetAllocator());
  document->AddMember("disk_info", DiskInfo::DebugString().c_str(),
      document->GetAllocator());
  document->AddMember("os_info", OsInfo::DebugString().c_str(),
      document->GetAllocator());
  document->AddMember("pid", getpid(), document->GetAllocator());
}

void Webserver::BuildArgumentMap(const string& args, ArgumentMap* output) {
  vector<string> arg_pairs;
  split(arg_pairs, args, is_any_of("&"));

  BOOST_FOREACH(const string& arg_pair, arg_pairs) {
    vector<string> key_value;
    split(key_value, arg_pair, is_any_of("="));
    if (key_value.empty()) continue;

    string key;
    if (!UrlDecode(key_value[0], &key)) continue;
    string value;
    if (!UrlDecode((key_value.size() >= 2 ? key_value[1] : ""), &value)) continue;
    to_lower(key);
    (*output)[key] = value;
  }
}

bool Webserver::IsSecure() const {
  return !FLAGS_webserver_certificate_file.empty();
}

Status Webserver::Start() {
  LOG(INFO) << "Starting webserver on " << http_address_;

  stringstream listening_spec;
  listening_spec << http_address_;

  if (IsSecure()) {
    LOG(INFO) << "Webserver: Enabling HTTPS support";
    // Squeasel makes sockets with 's' suffixes accept SSL traffic only
    listening_spec << "s";
  }
  string listening_str = listening_spec.str();
  vector<const char*> options;

  if (!FLAGS_webserver_doc_root.empty() && FLAGS_enable_webserver_doc_root) {
    LOG(INFO) << "Document root: " << FLAGS_webserver_doc_root;
    options.push_back("document_root");
    options.push_back(FLAGS_webserver_doc_root.c_str());
  } else {
    LOG(INFO)<< "Document root disabled";
  }

  if (IsSecure()) {
    options.push_back("ssl_certificate");
    options.push_back(FLAGS_webserver_certificate_file.c_str());
  }

  if (!FLAGS_webserver_authentication_domain.empty()) {
    options.push_back("authentication_domain");
    options.push_back(FLAGS_webserver_authentication_domain.c_str());
  }

  if (!FLAGS_webserver_password_file.empty()) {
    // Squeasel doesn't log anything if it can't stat the password file (but will if it
    // can't open it, which it tries to do during a request)
    if (!exists(FLAGS_webserver_password_file)) {
      stringstream ss;
      ss << "Webserver: Password file does not exist: " << FLAGS_webserver_password_file;
      return Status(ss.str());
    }
    LOG(INFO) << "Webserver: Password file is " << FLAGS_webserver_password_file;
    options.push_back("global_auth_file");
    options.push_back(FLAGS_webserver_password_file.c_str());
  }

  options.push_back("listening_ports");
  options.push_back(listening_str.c_str());

  // Options must be a NULL-terminated list
  options.push_back(NULL);

  // squeasel ignores SIGCHLD and we need it to run kinit. This means that since
  // squeasel does not reap its own children CGI programs must be avoided.
  // Save the signal handler so we can restore it after squeasel sets it to be ignored.
  sighandler_t sig_chld = signal(SIGCHLD, SIG_DFL);

  sq_callbacks callbacks;
  memset(&callbacks, 0, sizeof(callbacks));
  callbacks.begin_request = &Webserver::BeginRequestCallbackStatic;
  callbacks.log_message = &Webserver::LogMessageCallbackStatic;

  // To work around not being able to pass member functions as C callbacks, we store a
  // pointer to this server in the per-server state, and register a static method as the
  // default callback. That method unpacks the pointer to this and calls the real
  // callback.
  context_ = sq_start(&callbacks, reinterpret_cast<void*>(this), &options[0]);

  // Restore the child signal handler so wait() works properly.
  signal(SIGCHLD, sig_chld);

  if (context_ == NULL) {
    stringstream error_msg;
    error_msg << "Webserver: Could not start on address " << http_address_;
    return Status(error_msg.str());
  }

  UrlCallback default_callback =
      bind<void>(mem_fn(&Webserver::RootHandler), this, _1, _2);

  RegisterUrlCallback("/", "root.tmpl", default_callback);

  LOG(INFO) << "Webserver started";
  return Status::OK;
}

void Webserver::Stop() {
  if (context_ != NULL) {
    sq_stop(context_);
    context_ = NULL;
  }
}

void Webserver::GetCommonJson(Document* document) {
  DCHECK(document != NULL);
  Value obj(kObjectType);
  obj.AddMember("process-name", google::ProgramInvocationShortName(),
      document->GetAllocator());

  Value lst(kArrayType);
  BOOST_FOREACH(const UrlHandlerMap::value_type& handler, url_handlers_) {
    if (handler.second.is_on_nav_bar()) {
      Value obj(kObjectType);
      obj.AddMember("link", handler.first.c_str(), document->GetAllocator());
      obj.AddMember("title", handler.first.c_str(), document->GetAllocator());
      lst.PushBack(obj, document->GetAllocator());
    }
  }

  obj.AddMember("navbar", lst, document->GetAllocator());
  document->AddMember(COMMON_JSON_KEY, obj, document->GetAllocator());
}

int Webserver::LogMessageCallbackStatic(const struct sq_connection* connection,
    const char* message) {
  if (message != NULL) {
    LOG(INFO) << "Webserver: " << message;
  }
  return PROCESSING_COMPLETE;
}

int Webserver::BeginRequestCallbackStatic(struct sq_connection* connection) {
  struct sq_request_info* request_info = sq_get_request_info(connection);
  Webserver* instance = reinterpret_cast<Webserver*>(request_info->user_data);
  return instance->BeginRequestCallback(connection, request_info);
}

int Webserver::BeginRequestCallback(struct sq_connection* connection,
    struct sq_request_info* request_info) {
  if (!FLAGS_webserver_doc_root.empty() && FLAGS_enable_webserver_doc_root) {
    if (strncmp(DOC_FOLDER, request_info->uri, DOC_FOLDER_LEN) == 0) {
      VLOG(2) << "HTTP File access: " << request_info->uri;
      // Let Squeasel deal with this request; returning NULL will fall through
      // to the default handler which will serve files.
      return NOT_PROCESSED;
    }
  }
  shared_lock<shared_mutex> lock(url_handlers_lock_);
  UrlHandlerMap::const_iterator it = url_handlers_.find(request_info->uri);
  if (it == url_handlers_.end()) {
    sq_printf(connection, "HTTP/1.1 404 Not Found\r\n"
        "Content-Type: text/plain\r\n\r\n");
    sq_printf(connection, "No handler for URI %s\r\n\r\n", request_info->uri);
    return PROCESSING_COMPLETE;
  }

  map<string, string> arguments;
  if (request_info->query_string != NULL) {
    BuildArgumentMap(request_info->query_string, &arguments);
  }
  MonotonicStopWatch sw;
  sw.Start();

  Document document;
  document.SetObject();
  GetCommonJson(&document);

  bool send_html_headers = true;
  // The output of this page is accumulated into this stringstream.
  stringstream output;
  bool raw_json = (arguments.find("json") != arguments.end());
  if (raw_json) {
    // Callbacks may optionally be rendered as a text-only, pretty-printed Json document
    // (mostly for debugging or integration with third-party tools).
    StringBuffer strbuf;
    PrettyWriter<StringBuffer> writer(strbuf);
    it->second.callback()(arguments, &document);
    document.Accept(writer);
    output << strbuf.GetString();
    send_html_headers = false;
  } else {
    it->second.callback()(arguments, &document);
    if (arguments.find("raw") != arguments.end()) {
      document.AddMember(ENABLE_RAW_JSON_KEY, "true", document.GetAllocator());
    }
    if (document.HasMember(ENABLE_RAW_JSON_KEY)) {
      send_html_headers = false;
    }

    const string& full_template_path =
        Substitute("$0/$1/$2", FLAGS_webserver_doc_root, DOC_FOLDER,
            it->second.template_filename());
    ifstream tmpl(full_template_path.c_str());
    if (!tmpl.is_open()) {
      output << "Could not open template: " << full_template_path;
      send_html_headers = false;
    } else {
      stringstream buffer;
      buffer << tmpl.rdbuf();
      RenderTemplate(buffer.str(), Substitute("$0/", FLAGS_webserver_doc_root), document,
          &output);
    }
  }

  VLOG(3) << "Rendering page " << request_info->uri << " took "
          << PrettyPrinter::Print(sw.ElapsedTime(), TCounterType::CPU_TICKS);

  const string& str = output.str();
  // Without styling, render the page as plain text
  if (send_html_headers) {
    sq_printf(connection, "HTTP/1.1 200 OK\r\n"
        "Content-Type: text/html\r\n"
        "Content-Length: %d\r\n"
        "\r\n", (int)str.length());
  } else {
    sq_printf(connection, "HTTP/1.1 200 OK\r\n"
        "Content-Type: text/plain\r\n"
        "Content-Length: %d\r\n"
        "\r\n", (int)str.length());
  }

  // Make sure to use sq_write for printing the body; sq_printf truncates at 8kb
  sq_write(connection, str.c_str(), str.length());
  return PROCESSING_COMPLETE;
}

void Webserver::RegisterUrlCallback(const string& path,
    const string& template_filename, const UrlCallback& callback, bool is_on_nav_bar) {
  upgrade_lock<shared_mutex> lock(url_handlers_lock_);
  upgrade_to_unique_lock<shared_mutex> writer_lock(lock);
  DCHECK(url_handlers_.find(path) == url_handlers_.end())
      << "Duplicate Url handler for: " << path;

  url_handlers_.insert(
      make_pair(path, UrlHandler(callback, template_filename, is_on_nav_bar)));
}

}
<commit_msg>IMPALA-1151: Fix use-after-free in webserver<commit_after>// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "util/webserver.h"

#include <boost/algorithm/string.hpp>
#include <boost/bind.hpp>
#include <boost/filesystem.hpp>
#include <boost/foreach.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/mem_fn.hpp>
#include <boost/thread/locks.hpp>
#include <gutil/strings/substitute.h>
#include <map>
#include <fstream>
#include <stdio.h>
#include <signal.h>
#include <string>
#include <mustache/mustache.h>
#include <rapidjson/stringbuffer.h>
#include <rapidjson/prettywriter.h>

#include "common/logging.h"
#include "util/cpu-info.h"
#include "util/disk-info.h"
#include "util/mem-info.h"
#include "util/os-info.h"
#include "util/url-coding.h"
#include "util/debug-util.h"
#include "util/stopwatch.h"
#include "rpc/thrift-util.h"

using namespace std;
using namespace boost;
using namespace boost::filesystem;
using namespace google;
using namespace strings;
using namespace rapidjson;
using namespace mustache;

const char* GetDefaultDocumentRoot();

DEFINE_int32(webserver_port, 25000, "Port to start debug webserver on");
DEFINE_string(webserver_interface, "",
    "Interface to start debug webserver on. If blank, webserver binds to 0.0.0.0");
DEFINE_string(webserver_doc_root, GetDefaultDocumentRoot(),
    "Files under <webserver_doc_root>/www are accessible via the debug webserver. "
    "Defaults to $IMPALA_HOME, or if $IMPALA_HOME is not set, disables the document "
    "root");
DEFINE_bool(enable_webserver_doc_root, true,
    "If true, webserver may serve static files from the webserver_doc_root");

DEFINE_string(webserver_certificate_file, "",
    "The location of the debug webserver's SSL certificate file, in .pem format. If "
    "empty, webserver SSL support is not enabled");
DEFINE_string(webserver_authentication_domain, "",
    "Domain used for debug webserver authentication");
DEFINE_string(webserver_password_file, "",
    "(Optional) Location of .htpasswd file containing user names and hashed passwords for"
    " debug webserver authentication");

static const char* DOC_FOLDER = "/www/";
static const int DOC_FOLDER_LEN = strlen(DOC_FOLDER);

// Easy-to-read constants for Squeasel return codes
static const uint32_t PROCESSING_COMPLETE = 1;
static const uint32_t NOT_PROCESSED = 0;

// Standard keys in the json document sent to templates for rendering. Must be kept in
// sync with the templates themselves.
static const char* COMMON_JSON_KEY = "__common__";

// Returns $IMPALA_HOME if set, otherwise /tmp/impala_www
const char* GetDefaultDocumentRoot() {
  stringstream ss;
  char* impala_home = getenv("IMPALA_HOME");
  if (impala_home == NULL) {
    return ""; // Empty document root means don't serve static files
  } else {
    ss << impala_home;
  }

  // Deliberate memory leak, but this should be called exactly once.
  string* str = new string(ss.str());
  return str->c_str();
}

namespace impala {

const char* Webserver::ENABLE_RAW_JSON_KEY = "__raw__";

Webserver::Webserver() : context_(NULL) {
  http_address_ = MakeNetworkAddress(
      FLAGS_webserver_interface.empty() ? "0.0.0.0" : FLAGS_webserver_interface,
      FLAGS_webserver_port);
}

Webserver::Webserver(const int port) : context_(NULL) {
  http_address_ = MakeNetworkAddress("0.0.0.0", port);
}

Webserver::~Webserver() {
  Stop();
}

void Webserver::RootHandler(const Webserver::ArgumentMap& args, Document* document) {
  Value version(GetVersionString().c_str(), document->GetAllocator());
  document->AddMember("version", version, document->GetAllocator());
  Value cpu_info(CpuInfo::DebugString().c_str(), document->GetAllocator());
  document->AddMember("cpu_info", cpu_info, document->GetAllocator());
  Value mem_info(MemInfo::DebugString().c_str(), document->GetAllocator());
  document->AddMember("mem_info", mem_info, document->GetAllocator());
  Value disk_info(DiskInfo::DebugString().c_str(), document->GetAllocator());
  document->AddMember("disk_info", disk_info, document->GetAllocator());
  Value os_info(OsInfo::DebugString().c_str(), document->GetAllocator());
  document->AddMember("os_info", os_info, document->GetAllocator());
  document->AddMember("pid", getpid(), document->GetAllocator());
}

void Webserver::BuildArgumentMap(const string& args, ArgumentMap* output) {
  vector<string> arg_pairs;
  split(arg_pairs, args, is_any_of("&"));

  BOOST_FOREACH(const string& arg_pair, arg_pairs) {
    vector<string> key_value;
    split(key_value, arg_pair, is_any_of("="));
    if (key_value.empty()) continue;

    string key;
    if (!UrlDecode(key_value[0], &key)) continue;
    string value;
    if (!UrlDecode((key_value.size() >= 2 ? key_value[1] : ""), &value)) continue;
    to_lower(key);
    (*output)[key] = value;
  }
}

bool Webserver::IsSecure() const {
  return !FLAGS_webserver_certificate_file.empty();
}

Status Webserver::Start() {
  LOG(INFO) << "Starting webserver on " << http_address_;

  stringstream listening_spec;
  listening_spec << http_address_;

  if (IsSecure()) {
    LOG(INFO) << "Webserver: Enabling HTTPS support";
    // Squeasel makes sockets with 's' suffixes accept SSL traffic only
    listening_spec << "s";
  }
  string listening_str = listening_spec.str();
  vector<const char*> options;

  if (!FLAGS_webserver_doc_root.empty() && FLAGS_enable_webserver_doc_root) {
    LOG(INFO) << "Document root: " << FLAGS_webserver_doc_root;
    options.push_back("document_root");
    options.push_back(FLAGS_webserver_doc_root.c_str());
  } else {
    LOG(INFO)<< "Document root disabled";
  }

  if (IsSecure()) {
    options.push_back("ssl_certificate");
    options.push_back(FLAGS_webserver_certificate_file.c_str());
  }

  if (!FLAGS_webserver_authentication_domain.empty()) {
    options.push_back("authentication_domain");
    options.push_back(FLAGS_webserver_authentication_domain.c_str());
  }

  if (!FLAGS_webserver_password_file.empty()) {
    // Squeasel doesn't log anything if it can't stat the password file (but will if it
    // can't open it, which it tries to do during a request)
    if (!exists(FLAGS_webserver_password_file)) {
      stringstream ss;
      ss << "Webserver: Password file does not exist: " << FLAGS_webserver_password_file;
      return Status(ss.str());
    }
    LOG(INFO) << "Webserver: Password file is " << FLAGS_webserver_password_file;
    options.push_back("global_auth_file");
    options.push_back(FLAGS_webserver_password_file.c_str());
  }

  options.push_back("listening_ports");
  options.push_back(listening_str.c_str());

  // Options must be a NULL-terminated list
  options.push_back(NULL);

  // squeasel ignores SIGCHLD and we need it to run kinit. This means that since
  // squeasel does not reap its own children CGI programs must be avoided.
  // Save the signal handler so we can restore it after squeasel sets it to be ignored.
  sighandler_t sig_chld = signal(SIGCHLD, SIG_DFL);

  sq_callbacks callbacks;
  memset(&callbacks, 0, sizeof(callbacks));
  callbacks.begin_request = &Webserver::BeginRequestCallbackStatic;
  callbacks.log_message = &Webserver::LogMessageCallbackStatic;

  // To work around not being able to pass member functions as C callbacks, we store a
  // pointer to this server in the per-server state, and register a static method as the
  // default callback. That method unpacks the pointer to this and calls the real
  // callback.
  context_ = sq_start(&callbacks, reinterpret_cast<void*>(this), &options[0]);

  // Restore the child signal handler so wait() works properly.
  signal(SIGCHLD, sig_chld);

  if (context_ == NULL) {
    stringstream error_msg;
    error_msg << "Webserver: Could not start on address " << http_address_;
    return Status(error_msg.str());
  }

  UrlCallback default_callback =
      bind<void>(mem_fn(&Webserver::RootHandler), this, _1, _2);

  RegisterUrlCallback("/", "root.tmpl", default_callback);

  LOG(INFO) << "Webserver started";
  return Status::OK;
}

void Webserver::Stop() {
  if (context_ != NULL) {
    sq_stop(context_);
    context_ = NULL;
  }
}

void Webserver::GetCommonJson(Document* document) {
  DCHECK(document != NULL);
  Value obj(kObjectType);
  obj.AddMember("process-name", google::ProgramInvocationShortName(),
      document->GetAllocator());

  Value lst(kArrayType);
  BOOST_FOREACH(const UrlHandlerMap::value_type& handler, url_handlers_) {
    if (handler.second.is_on_nav_bar()) {
      Value obj(kObjectType);
      obj.AddMember("link", handler.first.c_str(), document->GetAllocator());
      obj.AddMember("title", handler.first.c_str(), document->GetAllocator());
      lst.PushBack(obj, document->GetAllocator());
    }
  }

  obj.AddMember("navbar", lst, document->GetAllocator());
  document->AddMember(COMMON_JSON_KEY, obj, document->GetAllocator());
}

int Webserver::LogMessageCallbackStatic(const struct sq_connection* connection,
    const char* message) {
  if (message != NULL) {
    LOG(INFO) << "Webserver: " << message;
  }
  return PROCESSING_COMPLETE;
}

int Webserver::BeginRequestCallbackStatic(struct sq_connection* connection) {
  struct sq_request_info* request_info = sq_get_request_info(connection);
  Webserver* instance = reinterpret_cast<Webserver*>(request_info->user_data);
  return instance->BeginRequestCallback(connection, request_info);
}

int Webserver::BeginRequestCallback(struct sq_connection* connection,
    struct sq_request_info* request_info) {
  if (!FLAGS_webserver_doc_root.empty() && FLAGS_enable_webserver_doc_root) {
    if (strncmp(DOC_FOLDER, request_info->uri, DOC_FOLDER_LEN) == 0) {
      VLOG(2) << "HTTP File access: " << request_info->uri;
      // Let Squeasel deal with this request; returning NULL will fall through
      // to the default handler which will serve files.
      return NOT_PROCESSED;
    }
  }
  shared_lock<shared_mutex> lock(url_handlers_lock_);
  UrlHandlerMap::const_iterator it = url_handlers_.find(request_info->uri);
  if (it == url_handlers_.end()) {
    sq_printf(connection, "HTTP/1.1 404 Not Found\r\n"
        "Content-Type: text/plain\r\n\r\n");
    sq_printf(connection, "No handler for URI %s\r\n\r\n", request_info->uri);
    return PROCESSING_COMPLETE;
  }

  map<string, string> arguments;
  if (request_info->query_string != NULL) {
    BuildArgumentMap(request_info->query_string, &arguments);
  }
  MonotonicStopWatch sw;
  sw.Start();

  Document document;
  document.SetObject();
  GetCommonJson(&document);

  bool send_html_headers = true;
  // The output of this page is accumulated into this stringstream.
  stringstream output;
  bool raw_json = (arguments.find("json") != arguments.end());
  if (raw_json) {
    // Callbacks may optionally be rendered as a text-only, pretty-printed Json document
    // (mostly for debugging or integration with third-party tools).
    StringBuffer strbuf;
    PrettyWriter<StringBuffer> writer(strbuf);
    it->second.callback()(arguments, &document);
    document.Accept(writer);
    output << strbuf.GetString();
    send_html_headers = false;
  } else {
    it->second.callback()(arguments, &document);
    if (arguments.find("raw") != arguments.end()) {
      document.AddMember(ENABLE_RAW_JSON_KEY, "true", document.GetAllocator());
    }
    if (document.HasMember(ENABLE_RAW_JSON_KEY)) {
      send_html_headers = false;
    }

    const string& full_template_path =
        Substitute("$0/$1/$2", FLAGS_webserver_doc_root, DOC_FOLDER,
            it->second.template_filename());
    ifstream tmpl(full_template_path.c_str());
    if (!tmpl.is_open()) {
      output << "Could not open template: " << full_template_path;
      send_html_headers = false;
    } else {
      stringstream buffer;
      buffer << tmpl.rdbuf();
      RenderTemplate(buffer.str(), Substitute("$0/", FLAGS_webserver_doc_root), document,
          &output);
    }
  }

  VLOG(3) << "Rendering page " << request_info->uri << " took "
          << PrettyPrinter::Print(sw.ElapsedTime(), TCounterType::CPU_TICKS);

  const string& str = output.str();
  // Without styling, render the page as plain text
  if (send_html_headers) {
    sq_printf(connection, "HTTP/1.1 200 OK\r\n"
        "Content-Type: text/html\r\n"
        "Content-Length: %d\r\n"
        "\r\n", (int)str.length());
  } else {
    sq_printf(connection, "HTTP/1.1 200 OK\r\n"
        "Content-Type: text/plain\r\n"
        "Content-Length: %d\r\n"
        "\r\n", (int)str.length());
  }

  // Make sure to use sq_write for printing the body; sq_printf truncates at 8kb
  sq_write(connection, str.c_str(), str.length());
  return PROCESSING_COMPLETE;
}

void Webserver::RegisterUrlCallback(const string& path,
    const string& template_filename, const UrlCallback& callback, bool is_on_nav_bar) {
  upgrade_lock<shared_mutex> lock(url_handlers_lock_);
  upgrade_to_unique_lock<shared_mutex> writer_lock(lock);
  DCHECK(url_handlers_.find(path) == url_handlers_.end())
      << "Duplicate Url handler for: " << path;

  url_handlers_.insert(
      make_pair(path, UrlHandler(callback, template_filename, is_on_nav_bar)));
}

}
<|endoftext|>


<commit_before>
#define QT_GUI_LIB

#include <stdlib.h>
#include <Qt/qobject.h>
#include <Qt/qwidget.h>
#include <Qt/qlist.h>
#include <Qt/qdebug.h>
#include <QtTest/QTest>
#include <phonon/Global>
#include <phonon/MediaObject>
#include <phonon/AudioOutput>
#include <phonon/VideoWidget>
#include <../globalconfig.h>

using namespace std;
using namespace Phonon::Experimental;

class AVCaptureTest : public QObject
{
    Q_OBJECT
    private slots:
        void initTestCase();
        void testAudioOutput();
        void testAudioCapture();
        void testVideoOutput();
        void testVideoCapture();
        void cleanupTestCase();
    private:
        Phonon::MediaObject *m_media;
        Phonon::AudioOutput *m_aout;
        Phonon::VideoWidget *m_vwid;
        Phonon::Experimental::GlobalConfig *m_pgc;
};

void AVCaptureTest::initTestCase()
{
    m_media = new Phonon::MediaObject();
    m_aout = new Phonon::AudioOutput(Phonon::VideoCategory, NULL);
    m_vwid = new Phonon::VideoWidget(NULL);

    Phonon::createPath(m_media, m_vwid);
    Phonon::createPath(m_media, m_aout);

    m_pgc = new Phonon::Experimental::GlobalConfig();
}

void AVCaptureTest::cleanupTestCase()
{
    delete m_pgc;
    delete m_aout;
    delete m_vwid;
    delete m_media;
}

void AVCaptureTest::testAudioOutput()
{
    // Write device indices
    QList<int> aoList = m_pgc->audioOutputDeviceListFor(Phonon::NoCategory);
    qDebug() << "Device list for audio output" << aoList;

    // Write device properties
    QHash<QByteArray, QVariant> info;
    int index;
    foreach(index, aoList)
    {
        info = m_pgc->audioOutputDeviceProperties(index);
        qDebug() << "Device properties for audio output" << index << ":";
        qDebug() << info;
        QVERIFY(!info.isEmpty());
    }
}

void AVCaptureTest::testAudioCapture()
{
    // Write device indices
    QList<int> acList = m_pgc->audioCaptureDeviceListFor(Phonon::NoCategory);
    qDebug() << "Device list for audio capture" << acList;

    // Write device properties
    QHash<QByteArray, QVariant> info;
    int index;
    foreach(index, acList)
    {
        info = m_pgc->audioCaptureDeviceProperties(index);
        qDebug() << "Device properties for audio capture" << index << ":";
        qDebug() << info;
        QVERIFY(!info.isEmpty());
    }
}

void AVCaptureTest::testVideoOutput()
{
    // Write device indices
    QList<int> vcList = m_pgc->videoCaptureDeviceListFor(Phonon::NoCategory);
    qDebug() << "Device list for video capture" << vcList;

    // Write device properties
    QHash<QByteArray, QVariant> info;
    int index;
    foreach(index, vcList)
    {
        info = m_pgc->videoCaptureDeviceProperties(index);
        qDebug() << "Device properties for video capture" << index << ":";
        qDebug() << info;
        QVERIFY(!info.isEmpty());
    }
}

void AVCaptureTest::testVideoCapture()
{
    // Write device indices
    QList<int> acList = m_pgc->videoCaptureDeviceListFor(Phonon::NoCategory);
    qDebug() << "Device list for video capture" << acList;

    // Write device properties
    QHash<QByteArray, QVariant> info;
    int index;
    foreach(index, acList)
    {
        info = m_pgc->videoCaptureDeviceProperties(index);
        qDebug() << "Device properties for video capture" << index << ":";
        qDebug() << info;
        QVERIFY(!info.isEmpty());
    }
}

QTEST_MAIN(AVCaptureTest)

#include "avcapturetest.moc"
<commit_msg>Removed useless test for video output device list<commit_after>
#define QT_GUI_LIB

#include <stdlib.h>
#include <Qt/qobject.h>
#include <Qt/qwidget.h>
#include <Qt/qlist.h>
#include <Qt/qdebug.h>
#include <QtTest/QTest>
#include <phonon/Global>
#include <phonon/MediaObject>
#include <phonon/AudioOutput>
#include <phonon/VideoWidget>
#include <../globalconfig.h>

using namespace std;
using namespace Phonon::Experimental;

class AVCaptureTest : public QObject
{
    Q_OBJECT
    private slots:
        void initTestCase();
        void testAudioOutput();
        void testAudioCapture();
        void testVideoCapture();
        void cleanupTestCase();
    private:
        Phonon::MediaObject *m_media;
        Phonon::AudioOutput *m_aout;
        Phonon::VideoWidget *m_vwid;
        Phonon::Experimental::GlobalConfig *m_pgc;
};

void AVCaptureTest::initTestCase()
{
    m_media = new Phonon::MediaObject();
    m_aout = new Phonon::AudioOutput(Phonon::VideoCategory, NULL);
    m_vwid = new Phonon::VideoWidget(NULL);

    Phonon::createPath(m_media, m_vwid);
    Phonon::createPath(m_media, m_aout);

    m_pgc = new Phonon::Experimental::GlobalConfig();
}

void AVCaptureTest::cleanupTestCase()
{
    delete m_pgc;
    delete m_aout;
    delete m_vwid;
    delete m_media;
}

void AVCaptureTest::testAudioOutput()
{
    // Write device indices
    QList<int> aoList = m_pgc->audioOutputDeviceListFor(Phonon::NoCategory);
    qDebug() << "Device list for audio output" << aoList;

    // Write device properties
    QHash<QByteArray, QVariant> info;
    int index;
    foreach(index, aoList)
    {
        info = m_pgc->audioOutputDeviceProperties(index);
        qDebug() << "Device properties for audio output" << index << ":";
        qDebug() << info;
        QVERIFY(!info.isEmpty());
    }
}

void AVCaptureTest::testAudioCapture()
{
    // Write device indices
    QList<int> acList = m_pgc->audioCaptureDeviceListFor(Phonon::NoCategory);
    qDebug() << "Device list for audio capture" << acList;

    // Write device properties
    QHash<QByteArray, QVariant> info;
    int index;
    foreach(index, acList)
    {
        info = m_pgc->audioCaptureDeviceProperties(index);
        qDebug() << "Device properties for audio capture" << index << ":";
        qDebug() << info;
        QVERIFY(!info.isEmpty());
    }
}

void AVCaptureTest::testVideoCapture()
{
    // Write device indices
    QList<int> acList = m_pgc->videoCaptureDeviceListFor(Phonon::NoCategory);
    qDebug() << "Device list for video capture" << acList;

    // Write device properties
    QHash<QByteArray, QVariant> info;
    int index;
    foreach(index, acList)
    {
        info = m_pgc->videoCaptureDeviceProperties(index);
        qDebug() << "Device properties for video capture" << index << ":";
        qDebug() << info;
        QVERIFY(!info.isEmpty());
    }
}

QTEST_MAIN(AVCaptureTest)

#include "avcapturetest.moc"
<|endoftext|>


<commit_before>#include <cstdio>
#include <cstdlib>
#include <cstring>

extern "C" {
#include "tabulated.h"
#include "clhash.h"
}

inline uint64_t RDTSC_START() {
    register unsigned cyc_high, cyc_low;
    __asm volatile("cpuid\n\t"
                   "rdtsc\n\t"
                   "mov %%edx, %0\n\t"
                   "mov %%eax, %1\n\t"
                   : "=r"(cyc_high), "=r"(cyc_low)::"%rax", "%rbx", "%rcx",
                     "%rdx");
    return ((uint64_t)cyc_high << 32) | cyc_low;
}

inline uint64_t RDTSC_FINAL() {
    register unsigned cyc_high, cyc_low;
    __asm volatile("rdtscp\n\t"
                   "mov %%edx, %0\n\t"
                   "mov %%eax, %1\n\t"
                   "cpuid\n\t"
                   : "=r"(cyc_high), "=r"(cyc_low)::"%rax", "%rbx", "%rcx",
                     "%rdx");
    return ((uint64_t)cyc_high << 32) | cyc_low;
}

static __attribute__((noinline)) uint64_t rdtsc_overhead_func(uint64_t dummy) {
    return dummy;
}

uint64_t global_rdtsc_overhead = (uint64_t)UINT64_MAX;

#define RDTSC_SET_OVERHEAD(test, repeat)                                       \
    do {                                                                       \
        uint64_t cycles_start, cycles_final, cycles_diff;                      \
        uint64_t min_diff = UINT64_MAX;                                        \
        for (int i = 0; i < repeat; i++) {                                     \
            __asm volatile("" ::: /* pretend to clobber */ "memory");          \
            cycles_start = RDTSC_START();                                      \
            test;                                                              \
            cycles_final = RDTSC_FINAL();                                      \
            cycles_diff = (cycles_final - cycles_start);                       \
            if (cycles_diff < min_diff)                                        \
                min_diff = cycles_diff;                                        \
        }                                                                      \
        global_rdtsc_overhead = min_diff;                                      \
        printf("rdtsc_overhead set to %d\n", (int)global_rdtsc_overhead);      \
    } while (0)

/*
 * Prints the best number of operations per cycle where
 * test is the function call,  repeat is the number of times we should repeat
 * and size is the
 * number of operations represented by test.
 */
#define BEST_TIME(test, pre, expected, repeat, size)                           \
    do {                                                                       \
        if (global_rdtsc_overhead == UINT64_MAX) {                             \
            RDTSC_SET_OVERHEAD(rdtsc_overhead_func(1), repeat);                \
        }                                                                      \
        printf("%s: ", #test);                                                 \
        fflush(NULL);                                                          \
        uint64_t cycles_start, cycles_final, cycles_diff;                      \
        uint64_t min_diff = (uint64_t)-1;                                      \
        int wrong_answer = 0;                                                  \
        for (int i = 0; i < repeat; i++) {                                     \
            __asm volatile("" ::: /* pretend to clobber */ "memory");          \
            pre;                                                               \
            cycles_start = RDTSC_START();                                      \
            if (test != expected)                                              \
                wrong_answer = 1;                                              \
            cycles_final = RDTSC_FINAL();                                      \
            cycles_diff = (cycles_final - cycles_start);                       \
            if (cycles_diff < min_diff)                                        \
                min_diff = cycles_diff;                                        \
        }                                                                      \
        min_diff -= global_rdtsc_overhead;                                     \
        uint64_t S = (uint64_t)size;                                           \
        float cycle_per_op = (min_diff) / (float)S;                            \
        printf("size = %d,  %.2f cycles per word ", size, cycle_per_op);       \
        if (wrong_answer)                                                      \
            printf(" [ERROR]");                                                \
        printf("\n");                                                          \
        fflush(NULL);                                                          \
    } while (0)

uint64_t demo_zobrist(uint64_t *array, uint32_t length, zobrist_t *k) {
    uint64_t sum = 0;
    for (uint32_t x = 0; x < length; ++x) {
        sum += zobrist(array[x], k);
    }
    return sum;
}

uint64_t demo_cl_linear(uint64_t *array, uint32_t length, cl_linear_t *k) {
    uint64_t sum = 0;
    for (uint32_t x = 0; x < length; ++x) {
        sum += cl_linear(array[x], k);
    }
    return sum;
}

uint64_t demo_cl_quadratic(uint64_t *array, uint32_t length,
                           cl_quadratic_t *k) {
    uint64_t sum = 0;
    for (uint32_t x = 0; x < length; ++x) {
        sum += cl_quadratic(array[x], k);
    }
    return sum;
}

uint64_t demo_cl_cubic(uint64_t *array, uint32_t length, cl_cubic_t *k) {
    uint64_t sum = 0;
    for (uint32_t x = 0; x < length; ++x) {
        sum += cl_cubic(array[x], k);
    }
    return sum;
}

void flush(void *b, size_t length) {
    char *B = (char *)b;
    for (uint32_t k = 0; k < length; k += 64) {
        __builtin_ia32_clflush(B + k);
    }
}

void basic(uint32_t length) {
    printf("Testing 64-bit hashing.\n");
    printf("We will construct an array of %d words (using %d bytes), to be "
           "hashed.\n",
           (int)length, (int)(length * sizeof(uint64_t)));
    printf("Keys are flushed at the beginning of each run.\n");
    cl_linear_t cl_lineark;
    cl_linear_init(&cl_lineark);

    cl_quadratic_t cl_quadratick;
    cl_quadratic_init(&cl_quadratick);

    cl_cubic_t cl_cubick;
    cl_cubic_init(&cl_cubick);

    zobrist_t zobristk;
    zobrist_init(&zobristk);

    printf("sizeof(cl_lineark) = %d, sizeof(cl_quadratick) = %d, "
           "sizeof(cl_cubick) = %d,  sizeof(zobristk) = %d \n",
           (int)sizeof(cl_lineark), (int)sizeof(cl_quadratick),
           (int)sizeof(cl_cubick), (int)sizeof(zobristk));

    uint64_t *array = (uint64_t *)malloc(sizeof(uint64_t) * length);
    for (uint32_t i = 0; i < length; ++i) {
        array[i] = get64rand();
    }
    uint64_t expected;
    uint32_t size = length;
    int repeat = 500;

    expected = demo_zobrist(array, length, &zobristk);
    BEST_TIME(demo_zobrist(array, length, &zobristk),
              flush(&zobristk, sizeof(zobristk)), expected, repeat, size);

    expected = demo_cl_linear(array, length, &cl_lineark);
    BEST_TIME(demo_cl_linear(array, length, &cl_lineark),
              flush(&cl_lineark, sizeof(cl_lineark)), expected, repeat, size);

    expected = demo_cl_quadratic(array, length, &cl_quadratick);
    BEST_TIME(demo_cl_quadratic(array, length, &cl_quadratick),
              flush(&cl_quadratick, sizeof(cl_quadratick)), expected, repeat,
              size);

    expected = demo_cl_cubic(array, length, &cl_cubick);
    BEST_TIME(demo_cl_cubic(array, length, &cl_cubick),
              flush(&cl_cubick, sizeof(cl_cubick)), expected, repeat, size);

    printf("zobrist is 3-wise ind., linear is 2-wise ind., quadratic is 3-wise "
           "ind., cubic is 4-wise ind.\n");

    free(array);
    printf("\n");
}

uint32_t demo_zobrist32(uint32_t *array, uint32_t length, zobrist32_t *k) {
    uint32_t sum = 0;
    for (uint32_t x = 0; x < length; ++x) {
        sum += zobrist32(array[x], k);
    }
    return sum;
}

uint32_t demo_cl_quadratic32(uint32_t *array, uint32_t length,
                             cl_quadratic_t *k) {
    uint32_t sum = 0;
    for (uint32_t x = 0; x < length; ++x) {
        sum += cl_quadratic32(array[x], k);
    }
    return sum;
}

uint32_t demo_cl_linear32(uint32_t *array, uint32_t length, cl_linear_t *k) {
    uint32_t sum = 0;
    for (uint32_t x = 0; x < length; ++x) {
        sum += cl_linear32(array[x], k);
    }
    return sum;
}

void basic32(uint32_t length) {
    printf("Testing 32-bit hashing.\n");
    printf("We will construct an array of %d words (using %d bytes), to be "
           "hashed.\n",
           (int)length, (int)(length * sizeof(uint32_t)));
    printf("Keys are flushed at the beginning of each run.\n");
    cl_quadratic_t cl_quadratick;
    cl_quadratic32_init(&cl_quadratick);

    cl_linear_t cl_lineark;
    cl_linear32_init(&cl_lineark);

    zobrist32_t zobristk;
    zobrist32_init(&zobristk);

    printf(" sizeof(cl_lineark) = %d, sizeof(cl_quadratick) = %d, "
           "sizeof(zobristk) = %d \n",
           (int)sizeof(cl_lineark), (int)sizeof(cl_quadratick),
           (int)sizeof(zobristk));

    uint32_t *array = (uint32_t *)malloc(sizeof(uint32_t) * length);
    for (uint32_t i = 0; i < length; ++i) {
        array[i] = get32rand();
    }
    uint32_t expected;
    uint32_t size = length;
    int repeat = 500;

    expected = demo_zobrist32(array, length, &zobristk);
    BEST_TIME(demo_zobrist32(array, length, &zobristk),
              flush(&zobristk, sizeof(zobristk)), expected, repeat, size);

    expected = demo_cl_linear32(array, length, &cl_lineark);
    BEST_TIME(demo_cl_linear32(array, length, &cl_lineark),
              flush(&cl_quadratick, sizeof(cl_lineark)), expected, repeat,
              size);

    expected = demo_cl_quadratic32(array, length, &cl_quadratick);
    BEST_TIME(demo_cl_quadratic32(array, length, &cl_quadratick),
              flush(&cl_quadratick, sizeof(cl_quadratick)), expected, repeat,
              size);

    printf("zobrist is 3-wise ind., linear is 2-wise ind., quadratic is 3-wise "
           "ind.\n");

    free(array);
    printf("\n");
}

int main() {
    printf("=======");
    basic(10);
    basic(20);
    basic(100);
    basic(1000);
    printf("=======");
    basic32(10);
    basic32(20);
    basic32(100);
    basic32(1000);

    printf("Large runs are beneficial to tabulation-based hashing because they "
           "amortize cache faults.\n");
}
<commit_msg>De-macro RDTSC_SET_OVERHEAD<commit_after>#include <cstdio>
#include <cstdlib>
#include <cstring>

extern "C" {
#include "tabulated.h"
#include "clhash.h"
}

inline uint64_t RDTSC_START() {
    register unsigned cyc_high, cyc_low;
    __asm volatile("cpuid\n\t"
                   "rdtsc\n\t"
                   "mov %%edx, %0\n\t"
                   "mov %%eax, %1\n\t"
                   : "=r"(cyc_high), "=r"(cyc_low)::"%rax", "%rbx", "%rcx",
                     "%rdx");
    return ((uint64_t)cyc_high << 32) | cyc_low;
}

inline uint64_t RDTSC_FINAL() {
    register unsigned cyc_high, cyc_low;
    __asm volatile("rdtscp\n\t"
                   "mov %%edx, %0\n\t"
                   "mov %%eax, %1\n\t"
                   "cpuid\n\t"
                   : "=r"(cyc_high), "=r"(cyc_low)::"%rax", "%rbx", "%rcx",
                     "%rdx");
    return ((uint64_t)cyc_high << 32) | cyc_low;
}

static __attribute__((noinline)) uint64_t rdtsc_overhead_func(uint64_t dummy) {
    return dummy;
}

uint64_t global_rdtsc_overhead = (uint64_t)UINT64_MAX;

void RDTSC_SET_OVERHEAD(int repeat) {
    uint64_t cycles_start, cycles_final, cycles_diff;
    uint64_t min_diff = UINT64_MAX;
    for (int i = 0; i < repeat; i++) {
        __asm volatile("" ::: /* pretend to clobber */ "memory");
        cycles_start = RDTSC_START();
        rdtsc_overhead_func(1);
        cycles_final = RDTSC_FINAL();
        cycles_diff = (cycles_final - cycles_start);
        if (cycles_diff < min_diff)
            min_diff = cycles_diff;
    }
    global_rdtsc_overhead = min_diff;
    printf("rdtsc_overhead set to %d\n", (int)global_rdtsc_overhead);
}

/*
 * Prints the best number of operations per cycle where
 * test is the function call,  repeat is the number of times we should repeat
 * and size is the
 * number of operations represented by test.
 */
#define BEST_TIME(test, pre, expected, repeat, size)                           \
    do {                                                                       \
        if (global_rdtsc_overhead == UINT64_MAX) {                             \
            RDTSC_SET_OVERHEAD(repeat);                                        \
        }                                                                      \
        printf("%s: ", #test);                                                 \
        fflush(NULL);                                                          \
        uint64_t cycles_start, cycles_final, cycles_diff;                      \
        uint64_t min_diff = (uint64_t)-1;                                      \
        int wrong_answer = 0;                                                  \
        for (int i = 0; i < repeat; i++) {                                     \
            __asm volatile("" ::: /* pretend to clobber */ "memory");          \
            pre;                                                               \
            cycles_start = RDTSC_START();                                      \
            if (test != expected)                                              \
                wrong_answer = 1;                                              \
            cycles_final = RDTSC_FINAL();                                      \
            cycles_diff = (cycles_final - cycles_start);                       \
            if (cycles_diff < min_diff)                                        \
                min_diff = cycles_diff;                                        \
        }                                                                      \
        min_diff -= global_rdtsc_overhead;                                     \
        uint64_t S = (uint64_t)size;                                           \
        float cycle_per_op = (min_diff) / (float)S;                            \
        printf("size = %d,  %.2f cycles per word ", size, cycle_per_op);       \
        if (wrong_answer)                                                      \
            printf(" [ERROR]");                                                \
        printf("\n");                                                          \
        fflush(NULL);                                                          \
    } while (0)

uint64_t demo_zobrist(uint64_t *array, uint32_t length, zobrist_t *k) {
    uint64_t sum = 0;
    for (uint32_t x = 0; x < length; ++x) {
        sum += zobrist(array[x], k);
    }
    return sum;
}

uint64_t demo_cl_linear(uint64_t *array, uint32_t length, cl_linear_t *k) {
    uint64_t sum = 0;
    for (uint32_t x = 0; x < length; ++x) {
        sum += cl_linear(array[x], k);
    }
    return sum;
}

uint64_t demo_cl_quadratic(uint64_t *array, uint32_t length,
                           cl_quadratic_t *k) {
    uint64_t sum = 0;
    for (uint32_t x = 0; x < length; ++x) {
        sum += cl_quadratic(array[x], k);
    }
    return sum;
}

uint64_t demo_cl_cubic(uint64_t *array, uint32_t length, cl_cubic_t *k) {
    uint64_t sum = 0;
    for (uint32_t x = 0; x < length; ++x) {
        sum += cl_cubic(array[x], k);
    }
    return sum;
}

void flush(void *b, size_t length) {
    char *B = (char *)b;
    for (uint32_t k = 0; k < length; k += 64) {
        __builtin_ia32_clflush(B + k);
    }
}

void basic(uint32_t length) {
    printf("Testing 64-bit hashing.\n");
    printf("We will construct an array of %d words (using %d bytes), to be "
           "hashed.\n",
           (int)length, (int)(length * sizeof(uint64_t)));
    printf("Keys are flushed at the beginning of each run.\n");
    cl_linear_t cl_lineark;
    cl_linear_init(&cl_lineark);

    cl_quadratic_t cl_quadratick;
    cl_quadratic_init(&cl_quadratick);

    cl_cubic_t cl_cubick;
    cl_cubic_init(&cl_cubick);

    zobrist_t zobristk;
    zobrist_init(&zobristk);

    printf("sizeof(cl_lineark) = %d, sizeof(cl_quadratick) = %d, "
           "sizeof(cl_cubick) = %d,  sizeof(zobristk) = %d \n",
           (int)sizeof(cl_lineark), (int)sizeof(cl_quadratick),
           (int)sizeof(cl_cubick), (int)sizeof(zobristk));

    uint64_t *array = (uint64_t *)malloc(sizeof(uint64_t) * length);
    for (uint32_t i = 0; i < length; ++i) {
        array[i] = get64rand();
    }
    uint64_t expected;
    uint32_t size = length;
    int repeat = 500;

    expected = demo_zobrist(array, length, &zobristk);
    BEST_TIME(demo_zobrist(array, length, &zobristk),
              flush(&zobristk, sizeof(zobristk)), expected, repeat, size);

    expected = demo_cl_linear(array, length, &cl_lineark);
    BEST_TIME(demo_cl_linear(array, length, &cl_lineark),
              flush(&cl_lineark, sizeof(cl_lineark)), expected, repeat, size);

    expected = demo_cl_quadratic(array, length, &cl_quadratick);
    BEST_TIME(demo_cl_quadratic(array, length, &cl_quadratick),
              flush(&cl_quadratick, sizeof(cl_quadratick)), expected, repeat,
              size);

    expected = demo_cl_cubic(array, length, &cl_cubick);
    BEST_TIME(demo_cl_cubic(array, length, &cl_cubick),
              flush(&cl_cubick, sizeof(cl_cubick)), expected, repeat, size);

    printf("zobrist is 3-wise ind., linear is 2-wise ind., quadratic is 3-wise "
           "ind., cubic is 4-wise ind.\n");

    free(array);
    printf("\n");
}

uint32_t demo_zobrist32(uint32_t *array, uint32_t length, zobrist32_t *k) {
    uint32_t sum = 0;
    for (uint32_t x = 0; x < length; ++x) {
        sum += zobrist32(array[x], k);
    }
    return sum;
}

uint32_t demo_cl_quadratic32(uint32_t *array, uint32_t length,
                             cl_quadratic_t *k) {
    uint32_t sum = 0;
    for (uint32_t x = 0; x < length; ++x) {
        sum += cl_quadratic32(array[x], k);
    }
    return sum;
}

uint32_t demo_cl_linear32(uint32_t *array, uint32_t length, cl_linear_t *k) {
    uint32_t sum = 0;
    for (uint32_t x = 0; x < length; ++x) {
        sum += cl_linear32(array[x], k);
    }
    return sum;
}

void basic32(uint32_t length) {
    printf("Testing 32-bit hashing.\n");
    printf("We will construct an array of %d words (using %d bytes), to be "
           "hashed.\n",
           (int)length, (int)(length * sizeof(uint32_t)));
    printf("Keys are flushed at the beginning of each run.\n");
    cl_quadratic_t cl_quadratick;
    cl_quadratic32_init(&cl_quadratick);

    cl_linear_t cl_lineark;
    cl_linear32_init(&cl_lineark);

    zobrist32_t zobristk;
    zobrist32_init(&zobristk);

    printf(" sizeof(cl_lineark) = %d, sizeof(cl_quadratick) = %d, "
           "sizeof(zobristk) = %d \n",
           (int)sizeof(cl_lineark), (int)sizeof(cl_quadratick),
           (int)sizeof(zobristk));

    uint32_t *array = (uint32_t *)malloc(sizeof(uint32_t) * length);
    for (uint32_t i = 0; i < length; ++i) {
        array[i] = get32rand();
    }
    uint32_t expected;
    uint32_t size = length;
    int repeat = 500;

    expected = demo_zobrist32(array, length, &zobristk);
    BEST_TIME(demo_zobrist32(array, length, &zobristk),
              flush(&zobristk, sizeof(zobristk)), expected, repeat, size);

    expected = demo_cl_linear32(array, length, &cl_lineark);
    BEST_TIME(demo_cl_linear32(array, length, &cl_lineark),
              flush(&cl_quadratick, sizeof(cl_lineark)), expected, repeat,
              size);

    expected = demo_cl_quadratic32(array, length, &cl_quadratick);
    BEST_TIME(demo_cl_quadratic32(array, length, &cl_quadratick),
              flush(&cl_quadratick, sizeof(cl_quadratick)), expected, repeat,
              size);

    printf("zobrist is 3-wise ind., linear is 2-wise ind., quadratic is 3-wise "
           "ind.\n");

    free(array);
    printf("\n");
}

int main() {
    printf("=======");
    basic(10);
    basic(20);
    basic(100);
    basic(1000);
    printf("=======");
    basic32(10);
    basic32(20);
    basic32(100);
    basic32(1000);

    printf("Large runs are beneficial to tabulation-based hashing because they "
           "amortize cache faults.\n");
}
<|endoftext|>


<commit_before>#pragma once

#include "Eigen/Dense"

namespace numerics {

template <int rows, typename Callable>
Eigen::Matrix<double, rows, 1> numerical_gradient(
    const Eigen::Matrix<double, rows, 1> &x,
    const Callable &fcn,
    const double feps = 1e-6) {
  static_assert(rows != Eigen::Dynamic, "No dynamic");
  using OutVec = Eigen::Matrix<double, rows, 1>;

  OutVec jac = OutVec::Zero();

  for (int k = 0; k < rows; ++k) {
    OutVec zero = OutVec::Zero();
    zero(k) = feps;

    jac(k) = (fcn(x + zero) - fcn(x - zero)) / (2.0 * feps);
  }
  return jac;
}

Eigen::VectorXd dynamic_numerical_gradient(
    const Eigen::VectorXd &x,
    const std::function<double(const Eigen::VectorXd &)> &fcn) {
  using OutVec = Eigen::VectorXd;
  constexpr double FEPS = 1e-6;
  constexpr double INV_2FEPS = 1.0 / (2.0 * 1e-6);

  OutVec jac = OutVec::Zero(x.rows());
  OutVec twiddle = OutVec::Zero(x.rows());
  for (int k = 0; k < x.rows(); ++k) {
    twiddle(k) = FEPS;
    jac(k) = (fcn(x + twiddle) - fcn(x - twiddle)) * INV_2FEPS;
    twiddle(k) = 0.0;
  }
  return jac;
}

template <int output_rows, int input_rows, typename Callable>
Eigen::Matrix<double, output_rows, input_rows> numerical_jacobian(
    const Eigen::Matrix<double, input_rows, 1> &x,
    const Callable &fcn,
    const double feps = 1e-6) {
  static_assert(output_rows != Eigen::Dynamic, "No dynamic");
  static_assert(input_rows != Eigen::Dynamic, "No dynamic");

  // df1/dx1    df1/dx2 ....
  // df2/dx1    df2/dx2 ....
  //    :          :
  //    :          :

  using OutVec = Eigen::Matrix<double, input_rows, 1>;
  using Jacobian = Eigen::Matrix<double, output_rows, input_rows>;
  Jacobian jac = Jacobian::Zero();

  for (int k = 0; k < input_rows; ++k) {
    OutVec zero = OutVec::Zero();
    zero(k) = feps;
    jac.col(k) = (fcn(x + zero) - fcn(x - zero)) / (2 * feps);
  }
  return jac;
}
}  // namespace numerics
<commit_msg>Better ascii art in numerics<commit_after>#pragma once

#include "Eigen/Dense"

namespace numerics {

template <int rows, typename Callable>
Eigen::Matrix<double, rows, 1> numerical_gradient(
    const Eigen::Matrix<double, rows, 1> &x,
    const Callable &fcn,
    const double feps = 1e-6) {
  static_assert(rows != Eigen::Dynamic, "No dynamic");
  using OutVec = Eigen::Matrix<double, rows, 1>;

  OutVec jac = OutVec::Zero();

  for (int k = 0; k < rows; ++k) {
    OutVec zero = OutVec::Zero();
    zero(k) = feps;

    jac(k) = (fcn(x + zero) - fcn(x - zero)) / (2.0 * feps);
  }
  return jac;
}

Eigen::VectorXd dynamic_numerical_gradient(
    const Eigen::VectorXd &x,
    const std::function<double(const Eigen::VectorXd &)> &fcn) {
  using OutVec = Eigen::VectorXd;
  constexpr double FEPS = 1e-6;
  constexpr double INV_2FEPS = 1.0 / (2.0 * FEPS);

  OutVec jac = OutVec::Zero(x.rows());
  OutVec twiddle = OutVec::Zero(x.rows());
  for (int k = 0; k < x.rows(); ++k) {
    twiddle(k) = FEPS;
    jac(k) = (fcn(x + twiddle) - fcn(x - twiddle)) * INV_2FEPS;
    twiddle(k) = 0.0;
  }
  return jac;
}

template <int output_rows, int input_rows, typename Callable>
Eigen::Matrix<double, output_rows, input_rows> numerical_jacobian(
    const Eigen::Matrix<double, input_rows, 1> &x,
    const Callable &fcn,
    const double feps = 1e-6) {
  static_assert(output_rows != Eigen::Dynamic, "No dynamic");
  static_assert(input_rows != Eigen::Dynamic, "No dynamic");

  // df1/dx1    df1/dx2 .. ..    df1/dxm
  // df2/dx1    df2/dx2 .. ..    df2/dxm
  //    :          :     ..
  //    :          :           ..
  // dfn/dx1    dfn/dx2              ..

  using OutVec = Eigen::Matrix<double, input_rows, 1>;
  using Jacobian = Eigen::Matrix<double, output_rows, input_rows>;
  Jacobian jac = Jacobian::Zero();

  for (int k = 0; k < input_rows; ++k) {
    OutVec zero = OutVec::Zero();
    zero(k) = feps;
    jac.col(k) = (fcn(x + zero) - fcn(x - zero)) / (2 * feps);
  }
  return jac;
}
}  // namespace numerics
<|endoftext|>


<commit_before>#include <iostream>
#include <ctime>
#include <fstream>
#include "../src/MPSegment.hpp"
#include "../src/HMMSegment.hpp"
#include "../src/MixSegment.hpp"
#include "../src/KeywordExtractor.hpp"

using namespace CppJieba;

void cut(size_t times = 20)
{
    MixSegment seg("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8");
    vector<string> res;
    string doc;
    ifstream ifs("../test/testdata/weicheng.utf8");
    assert(ifs);
    doc << ifs;
    long beginTime = clock();
    for(size_t i = 0; i < times; i ++)
    {
        printf("process [%3.0lf %%]\r", 100.0*(i+1)/times);
        fflush(stdout);
        res.clear();
        seg.cut(doc, res);
    }
    long endTime = clock();
    printf("\ncut: [%.3lf seconds]time consumed.\n", double(endTime - beginTime)/CLOCKS_PER_SEC);
}

void extract(size_t times = 400)
{
    KeywordExtractor extractor("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8", "../dict/idf.utf8", "../dict/stop_words.utf8");
    vector<string> words;
    string doc;
    ifstream ifs("../test/testdata/review.100");
    assert(ifs);
    doc << ifs;
    long beginTime = clock();
    for(size_t i = 0; i < times; i ++)
    {
        printf("process [%3.0lf %%]\r", 100.0*(i+1)/times);
        fflush(stdout);
        words.clear();
        extractor.extract(doc, words, 5);
    }
    long endTime = clock();
    printf("\nextract: [%.3lf seconds]time consumed.\n", double(endTime - beginTime)/CLOCKS_PER_SEC);
}

int main(int argc, char ** argv)
{
    cut();
    extract();
    return EXIT_SUCCESS;
}
<commit_msg>make load_test test time longer<commit_after>#include <iostream>
#include <ctime>
#include <fstream>
#include "../src/MPSegment.hpp"
#include "../src/HMMSegment.hpp"
#include "../src/MixSegment.hpp"
#include "../src/KeywordExtractor.hpp"

using namespace CppJieba;

void cut(size_t times = 50)
{
    MixSegment seg("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8");
    vector<string> res;
    string doc;
    ifstream ifs("../test/testdata/weicheng.utf8");
    assert(ifs);
    doc << ifs;
    long beginTime = clock();
    for(size_t i = 0; i < times; i ++)
    {
        printf("process [%3.0lf %%]\r", 100.0*(i+1)/times);
        fflush(stdout);
        res.clear();
        seg.cut(doc, res);
    }
    long endTime = clock();
    printf("\ncut: [%.3lf seconds]time consumed.\n", double(endTime - beginTime)/CLOCKS_PER_SEC);
}

void extract(size_t times = 400)
{
    KeywordExtractor extractor("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8", "../dict/idf.utf8", "../dict/stop_words.utf8");
    vector<string> words;
    string doc;
    ifstream ifs("../test/testdata/review.100");
    assert(ifs);
    doc << ifs;
    long beginTime = clock();
    for(size_t i = 0; i < times; i ++)
    {
        printf("process [%3.0lf %%]\r", 100.0*(i+1)/times);
        fflush(stdout);
        words.clear();
        extractor.extract(doc, words, 5);
    }
    long endTime = clock();
    printf("\nextract: [%.3lf seconds]time consumed.\n", double(endTime - beginTime)/CLOCKS_PER_SEC);
}

int main(int argc, char ** argv)
{
    cut();
    extract();
    return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>#include "bench.h"

#include <stdlib.h>
#include <stdint.h>
#include <stdio.h>
#include <sys/time.h>

#include "box/box.h"
#include "request.h"
#include "tuple.h"
#include "port.h"
#include "fiber.h"

#if defined(MSGPUCK_H_INCLUDED)
#define MASTER 1
#include "iproto_constants.h"
#endif

enum { REQUEST_BODY_MAXLEN = 1024 };

/* {{{ Utils */

#if !defined(MASTER)
void
assert_fail(const char *assertion, const char *file, unsigned int line,
	    const char *function)
{
	(void)function;
	fflush(NULL);
	printf("assert: %s:%d %s\n", file, line, assertion);
	exit(1);
}
#endif /* STABLE */

double
nowtime(void)
{
	struct timeval t;
	gettimeofday(&t, NULL);
	return t.tv_sec + t.tv_usec * 1e-6;
}

void
randstr(char *out, size_t len)
{
	static const char alphanum[] =
			"0123456789-_"
			"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
			"abcdefghijklmnopqrstuvwxyz";
	static const size_t alen = sizeof(alphanum);
	for (size_t i = 0; i < len; ++i)
		out[i] = alphanum[rand() % (alen - 1)];
	out[len] = '\0';
}

/* }}} */

/* {{{ Generators */

char *
gen_uint(char *r, const struct keygen_params *params)
{
	(void) params;
#if defined(MASTER)
	r = mp_encode_array(r, 1);
	r = mp_encode_uint(r, rand());
#else /* STABLE */
	r = pack_u32(r, 1);
	r = pack_varint32(r, sizeof(uint32_t));
	r = pack_u32(r, rand());
#endif
	return r;
}

API_EXPORT char *
gen_str(char *r, const struct keygen_params *params)
{
	char buf[params->len + 1];
#if defined(MASTER)
	r = mp_encode_array(r, 1);
	randstr(buf, params->len);
	r = mp_encode_str(r, buf, params->len);
#else /* STABLE */
	r = pack_u32(r, 1);
	randstr(buf, params->len);
	r = pack_lstr(r, buf, params->len);
#endif
	return r;
}

char *
gen_uint_uint(char *r, const struct keygen_params *params)
{
	(void) params;
#if defined(MASTER)
	r = mp_encode_array(r, 2);
	r = mp_encode_uint(r, rand());
	r = mp_encode_uint(r, rand());
#else /* STABLE */
	r = pack_u32(r, 2);
	r = pack_varint32(r, sizeof(uint32_t));
	r = pack_u32(r, rand());
	r = pack_varint32(r, sizeof(uint32_t));
	r = pack_u32(r, rand());
#endif
	return r;
}

API_EXPORT char *
gen_str_str(char *r, const struct keygen_params *params)
{
	char buf[params->len + 1];
#if defined(MASTER)
	r = mp_encode_array(r, 2);
	randstr(buf, params->len);
	r = mp_encode_str(r, buf, params->len);
	randstr(buf, params->len);
	r = mp_encode_str(r, buf, params->len);
#else /* STABLE */
	r = pack_u32(r, 2);
	randstr(buf, params->len);
	r = pack_lstr(r, buf, params->len);
	randstr(buf, params->len);
	r = pack_lstr(r, buf, params->len);
#endif
	return r;
}

char *
gen_uint_str(char *r, const struct keygen_params *params)
{
	char buf[params->len + 1];
#if defined(MASTER)
	r = mp_encode_array(r, 2);
	r = mp_encode_uint(r, rand());
	randstr(buf, params->len);
	r = mp_encode_str(r, buf, params->len);
#else /* STABLE */
	r = pack_u32(r, 2);
	r = pack_varint32(r, sizeof(uint32_t));
	r = pack_u32(r, rand());
	randstr(buf, params->len);
	r = pack_lstr(r, buf, params->len);
#endif
	return r;
}

char *
gen_str_uint(char *r, const struct keygen_params *params)
{
	char buf[params->len + 1];
#if defined(MASTER)
	r = mp_encode_array(r, 2);
	randstr(buf, params->len);
	r = mp_encode_str(r, buf, params->len);
	r = mp_encode_uint(r, rand());
#else /* STABLE */
	r = pack_u32(r, 2);
	randstr(buf, params->len);
	r = pack_lstr(r, buf, params->len);
	r = pack_varint32(r, sizeof(uint32_t));
	r = pack_u32(r, rand());
#endif
	return r;
}

/* }}} */

/* {{{ Tests */

struct port_bench
{
	struct port_vtab *vtab;
	size_t count;
};

#if defined(MASTER)
static void
port_bench_add_tuple(struct port *port, struct tuple *tuple)
{
	(void) tuple;
	struct port_bench *port_bench = (struct port_bench *) port;
	port_bench->count++;
}
#else /* STABLE */
static void
port_bench_add_tuple(struct port *port, struct tuple *tuple, uint32_t flags)
{
	(void) tuple;
	(void) flags;
	struct port_bench *port_bench = (struct port_bench *) port;
	port_bench->count++;
}
#endif

static struct port_vtab port_bench_vtab = {
	port_bench_add_tuple,
	null_port_eof,
};

void
test_keys(const struct test_params *params)
{
	char reqdata[REQUEST_BODY_MAXLEN];

	for (uint32_t i = 0; i < params->count; i++) {
		char *r = reqdata;
		params->keygen(r, params->keygen_params);
	}
}

void
test_selects(const struct test_params *params)
{
	char reqdata[REQUEST_BODY_MAXLEN];
	struct port_bench port = { &port_bench_vtab, 0 };

	for (uint32_t i = 0; i < params->count; i++) {
		port.count = 0;
		char *r = reqdata;
#if defined(MASTER)
		struct request request;
		request_create(&request, IPROTO_SELECT);
		request.space_id = params->space_id;
		request.key = r;
		request.key_end = params->keygen(r, params->keygen_params);
		box_process((struct port *) &port, &request);
#else /* STABLE */
		r = pack_u32(r, params->space_id); // space
		r = pack_u32(r, 0); // index
		r = pack_u32(r, 0); // offset
		r = pack_u32(r, 4294967295); // limit
		r = pack_u32(r, 1); // key count
		r = params->keygen(r, params->keygen_params);
		box_process((struct port *) &port, SELECT, reqdata, r - reqdata);
#endif
		assert(port.count == 1);
		fiber_gc();
	}
}

void
test_replaces(const struct test_params *params)
{
	char reqdata[REQUEST_BODY_MAXLEN];
	struct port_bench port = { &port_bench_vtab, 0 };

	for (uint32_t i = 0; i < params->count; i++) {
		port.count = 0;
		char *r = reqdata;
#if defined(MASTER)
		struct request request;
		request_create(&request, IPROTO_REPLACE);
		request.space_id = params->space_id;
		request.tuple = r;
		request.tuple_end = params->keygen(r, params->keygen_params);
		box_process((struct port *) &port, &request);
#else /* STABLE */
		r = pack_u32(r, params->space_id); // space
		r = pack_u32(r, 0); // flags
		r = params->keygen(r, params->keygen_params);
		box_process((struct port *) &port, REPLACE, reqdata, r - reqdata);
#endif
		fiber_gc();
	}
}

void
test_deletes(const struct test_params *params)
{
	char reqdata[REQUEST_BODY_MAXLEN];
	struct port_bench port = { &port_bench_vtab, 0 };

	for (uint32_t i = 0; i < params->count; i++) {
		char *r = reqdata;
		port.count = 0;
#if defined(MASTER)
		struct request request;
		request_create(&request, IPROTO_DELETE);
		request.space_id = params->space_id;
		request.key = r;
		request.key_end = params->keygen(r, params->keygen_params);
		box_process((struct port *) &port, &request);
#else /* STABLE */
		r = pack_u32(r, params->space_id);
		r = pack_u32(r, 0); /* flags */
		r = params->keygen(r, params->keygen_params);
		box_process((struct port *) &port, DELETE, reqdata, r - reqdata);
#endif
		fiber_gc();
	}
}

/* }}} */

<commit_msg>Fix limit<commit_after>#include "bench.h"

#include <stdlib.h>
#include <stdint.h>
#include <stdio.h>
#include <sys/time.h>

#include "box/box.h"
#include "request.h"
#include "tuple.h"
#include "port.h"
#include "fiber.h"

#if defined(MSGPUCK_H_INCLUDED)
#define MASTER 1
#include "iproto_constants.h"
#endif

enum { REQUEST_BODY_MAXLEN = 1024 };

/* {{{ Utils */

#if !defined(MASTER)
void
assert_fail(const char *assertion, const char *file, unsigned int line,
	    const char *function)
{
	(void)function;
	fflush(NULL);
	printf("assert: %s:%d %s\n", file, line, assertion);
	exit(1);
}
#endif /* STABLE */

double
nowtime(void)
{
	struct timeval t;
	gettimeofday(&t, NULL);
	return t.tv_sec + t.tv_usec * 1e-6;
}

void
randstr(char *out, size_t len)
{
	static const char alphanum[] =
			"0123456789-_"
			"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
			"abcdefghijklmnopqrstuvwxyz";
	static const size_t alen = sizeof(alphanum);
	for (size_t i = 0; i < len; ++i)
		out[i] = alphanum[rand() % (alen - 1)];
	out[len] = '\0';
}

/* }}} */

/* {{{ Generators */

char *
gen_uint(char *r, const struct keygen_params *params)
{
	(void) params;
#if defined(MASTER)
	r = mp_encode_array(r, 1);
	r = mp_encode_uint(r, rand());
#else /* STABLE */
	r = pack_u32(r, 1);
	r = pack_varint32(r, sizeof(uint32_t));
	r = pack_u32(r, rand());
#endif
	return r;
}

API_EXPORT char *
gen_str(char *r, const struct keygen_params *params)
{
	char buf[params->len + 1];
#if defined(MASTER)
	r = mp_encode_array(r, 1);
	randstr(buf, params->len);
	r = mp_encode_str(r, buf, params->len);
#else /* STABLE */
	r = pack_u32(r, 1);
	randstr(buf, params->len);
	r = pack_lstr(r, buf, params->len);
#endif
	return r;
}

char *
gen_uint_uint(char *r, const struct keygen_params *params)
{
	(void) params;
#if defined(MASTER)
	r = mp_encode_array(r, 2);
	r = mp_encode_uint(r, rand());
	r = mp_encode_uint(r, rand());
#else /* STABLE */
	r = pack_u32(r, 2);
	r = pack_varint32(r, sizeof(uint32_t));
	r = pack_u32(r, rand());
	r = pack_varint32(r, sizeof(uint32_t));
	r = pack_u32(r, rand());
#endif
	return r;
}

API_EXPORT char *
gen_str_str(char *r, const struct keygen_params *params)
{
	char buf[params->len + 1];
#if defined(MASTER)
	r = mp_encode_array(r, 2);
	randstr(buf, params->len);
	r = mp_encode_str(r, buf, params->len);
	randstr(buf, params->len);
	r = mp_encode_str(r, buf, params->len);
#else /* STABLE */
	r = pack_u32(r, 2);
	randstr(buf, params->len);
	r = pack_lstr(r, buf, params->len);
	randstr(buf, params->len);
	r = pack_lstr(r, buf, params->len);
#endif
	return r;
}

char *
gen_uint_str(char *r, const struct keygen_params *params)
{
	char buf[params->len + 1];
#if defined(MASTER)
	r = mp_encode_array(r, 2);
	r = mp_encode_uint(r, rand());
	randstr(buf, params->len);
	r = mp_encode_str(r, buf, params->len);
#else /* STABLE */
	r = pack_u32(r, 2);
	r = pack_varint32(r, sizeof(uint32_t));
	r = pack_u32(r, rand());
	randstr(buf, params->len);
	r = pack_lstr(r, buf, params->len);
#endif
	return r;
}

char *
gen_str_uint(char *r, const struct keygen_params *params)
{
	char buf[params->len + 1];
#if defined(MASTER)
	r = mp_encode_array(r, 2);
	randstr(buf, params->len);
	r = mp_encode_str(r, buf, params->len);
	r = mp_encode_uint(r, rand());
#else /* STABLE */
	r = pack_u32(r, 2);
	randstr(buf, params->len);
	r = pack_lstr(r, buf, params->len);
	r = pack_varint32(r, sizeof(uint32_t));
	r = pack_u32(r, rand());
#endif
	return r;
}

/* }}} */

/* {{{ Tests */

struct port_bench
{
	struct port_vtab *vtab;
	size_t count;
};

#if defined(MASTER)
static void
port_bench_add_tuple(struct port *port, struct tuple *tuple)
{
	(void) tuple;
	struct port_bench *port_bench = (struct port_bench *) port;
	port_bench->count++;
}
#else /* STABLE */
static void
port_bench_add_tuple(struct port *port, struct tuple *tuple, uint32_t flags)
{
	(void) tuple;
	(void) flags;
	struct port_bench *port_bench = (struct port_bench *) port;
	port_bench->count++;
}
#endif

static struct port_vtab port_bench_vtab = {
	port_bench_add_tuple,
	null_port_eof,
};

void
test_keys(const struct test_params *params)
{
	char reqdata[REQUEST_BODY_MAXLEN];

	for (uint32_t i = 0; i < params->count; i++) {
		char *r = reqdata;
		params->keygen(r, params->keygen_params);
	}
}

void
test_selects(const struct test_params *params)
{
	char reqdata[REQUEST_BODY_MAXLEN];
	struct port_bench port = { &port_bench_vtab, 0 };

	for (uint32_t i = 0; i < params->count; i++) {
		port.count = 0;
		char *r = reqdata;
#if defined(MASTER)
		struct request request;
		request_create(&request, IPROTO_SELECT);
		request.space_id = params->space_id;
		request.limit = 1;
		request.key = r;
		request.key_end = params->keygen(r, params->keygen_params);
		box_process((struct port *) &port, &request);
#else /* STABLE */
		r = pack_u32(r, params->space_id); // space
		r = pack_u32(r, 0); // index
		r = pack_u32(r, 0); // offset
		r = pack_u32(r, 4294967295); // limit
		r = pack_u32(r, 1); // key count
		r = params->keygen(r, params->keygen_params);
		box_process((struct port *) &port, SELECT, reqdata, r - reqdata);
#endif
		assert(port.count == 1);
		fiber_gc();
	}
}

void
test_replaces(const struct test_params *params)
{
	char reqdata[REQUEST_BODY_MAXLEN];
	struct port_bench port = { &port_bench_vtab, 0 };

	for (uint32_t i = 0; i < params->count; i++) {
		port.count = 0;
		char *r = reqdata;
#if defined(MASTER)
		struct request request;
		request_create(&request, IPROTO_REPLACE);
		request.space_id = params->space_id;
		request.tuple = r;
		request.tuple_end = params->keygen(r, params->keygen_params);
		box_process((struct port *) &port, &request);
#else /* STABLE */
		r = pack_u32(r, params->space_id); // space
		r = pack_u32(r, 0); // flags
		r = params->keygen(r, params->keygen_params);
		box_process((struct port *) &port, REPLACE, reqdata, r - reqdata);
#endif
		fiber_gc();
	}
}

void
test_deletes(const struct test_params *params)
{
	char reqdata[REQUEST_BODY_MAXLEN];
	struct port_bench port = { &port_bench_vtab, 0 };

	for (uint32_t i = 0; i < params->count; i++) {
		char *r = reqdata;
		port.count = 0;
#if defined(MASTER)
		struct request request;
		request_create(&request, IPROTO_DELETE);
		request.space_id = params->space_id;
		request.key = r;
		request.key_end = params->keygen(r, params->keygen_params);
		box_process((struct port *) &port, &request);
#else /* STABLE */
		r = pack_u32(r, params->space_id);
		r = pack_u32(r, 0); /* flags */
		r = params->keygen(r, params->keygen_params);
		box_process((struct port *) &port, DELETE, reqdata, r - reqdata);
#endif
		fiber_gc();
	}
}

/* }}} */

<|endoftext|>