<commit_before>// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "paddle/fluid/operators/dot_op.h"

namespace paddle {
namespace operators {

class DotOp : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

  void InferShape(framework::InferShapeContext* ctx) const override {
    PADDLE_ENFORCE_EQ(true, ctx->HasInput("X"),
                      platform::errors::PreconditionNotMet(
                          "Input(X) of DotOp should not be null."));
    PADDLE_ENFORCE_EQ(true, ctx->HasInput("Y"),
                      platform::errors::PreconditionNotMet(
                          "Input(Y) of DotOp should not be null."));
    PADDLE_ENFORCE_EQ(true, ctx->HasOutput("Out"),
                      platform::errors::PreconditionNotMet(
                          "Output(Out) of DotOp should not be null."));

    auto x_dims = ctx->GetInputDim("X");
    auto x_rank = static_cast<size_t>(x_dims.size());
    PADDLE_ENFORCE_EQ(true, 1 == x_rank || 2 == x_rank,
                      platform::errors::PreconditionNotMet(
                          "ShapeError: The dimensions of input tensor X (%s) "
                          "should be 1 or 2",
                          x_dims.to_str()));

    auto y_dims = ctx->GetInputDim("Y");
    PADDLE_ENFORCE_EQ(
        true, x_rank == (size_t)y_dims.size(),
        platform::errors::PreconditionNotMet(
            "ShapeError: The shape of input tensor Y: %s should match with "
            "input tenosr X: %s",
            y_dims.to_str(), x_dims.to_str()));
    bool shape_match = true;
    for (size_t i = 0; i < x_rank; ++i) {
      if (x_dims[i] != y_dims[i]) {
        shape_match = false;
        break;
      }
    }

    PADDLE_ENFORCE_EQ(true, shape_match,
                      platform::errors::PreconditionNotMet(
                          "ShapeError: The shape of input tensor X: %s should "
                          "be exactly the same "
                          "with input tensor Y: %s",
                          x_dims.to_str(), y_dims.to_str()));
    auto dims = vectorize(x_dims);
    dims[dims.size() - 1] = 1;
    ctx->SetOutputDim("Out", phi::make_ddim(dims));
  }

  framework::OpKernelType GetExpectedKernelType(
      const framework::ExecutionContext& ctx) const override {
    return framework::OpKernelType(
        OperatorWithKernel::IndicateVarDataType(ctx, "X"), ctx.GetPlace());
  }
};

class DotOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
  void Make() final {
    AddInput("X", "(Tensor) The first input tensor. ");
    AddInput("Y", "(Tensor) The second input tensor. ");
    AddOutput("Out", "(Tensor) The result tensor.");
    AddComment("");
  }
};

class DotGradOp : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

  void InferShape(framework::InferShapeContext* ctx) const override {
    PADDLE_ENFORCE_EQ(
        true, ctx->HasInput("X"),
        platform::errors::PreconditionNotMet("Input(X) should not be null."));
    PADDLE_ENFORCE_EQ(
        true, ctx->HasInput("Y"),
        platform::errors::PreconditionNotMet("Input(Y) should not be null."));
    PADDLE_ENFORCE_EQ(true, ctx->HasInput(framework::GradVarName("Out")),
                      platform::errors::PreconditionNotMet(
                          "Input(Out@GRAD) should not be null."));

    auto x_grad_name = framework::GradVarName("X");
    auto y_grad_name = framework::GradVarName("Y");
    if (ctx->HasOutput(x_grad_name)) {
      ctx->ShareDim("X", /*->*/ x_grad_name);
      ctx->ShareLoD("X", /*->*/ x_grad_name);
    }
    if (ctx->HasOutput(y_grad_name)) {
      ctx->ShareDim("Y", /*->*/ y_grad_name);
      ctx->ShareLoD("Y", /*->*/ y_grad_name);
    }
  }

 protected:
  framework::OpKernelType GetExpectedKernelType(
      const framework::ExecutionContext& ctx) const override {
    return framework::OpKernelType(OperatorWithKernel::IndicateVarDataType(
                                       ctx, framework::GradVarName("Out")),
                                   ctx.GetPlace());
  }
};

template <typename T>
class DotOpGradMaker : public framework::SingleGradOpMaker<T> {
 public:
  using framework::SingleGradOpMaker<T>::SingleGradOpMaker;

 protected:
  void Apply(GradOpPtr<T> op) const override {
    op->SetType("dot_grad");

    op->SetInput("X", this->Input("X"));
    op->SetInput("Y", this->Input("Y"));
    op->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out"));
    op->SetAttrMap(this->Attrs());
    op->SetOutput(framework::GradVarName("X"), this->InputGrad("X"));
    op->SetOutput(framework::GradVarName("Y"), this->InputGrad("Y"));
  }
};

}  // namespace operators
}  // namespace paddle

namespace ops = paddle::operators;

REGISTER_OPERATOR(dot, ops::DotOp, ops::DotOpMaker,
                  ops::DotOpGradMaker<paddle::framework::OpDesc>,
                  ops::DotOpGradMaker<paddle::imperative::OpBase>);

REGISTER_OPERATOR(dot_grad, ops::DotGradOp);

REGISTER_OP_CPU_KERNEL(
    dot, ops::DotKernel<paddle::platform::CPUDeviceContext, float>,
    ops::DotKernel<paddle::platform::CPUDeviceContext, double>,
    ops::DotKernel<paddle::platform::CPUDeviceContext, int>,
    ops::DotKernel<paddle::platform::CPUDeviceContext, int64_t>,
    ops::DotKernel<paddle::platform::CPUDeviceContext,
                   paddle::platform::complex<float>>,
    ops::DotKernel<paddle::platform::CPUDeviceContext,
                   paddle::platform::complex<double>>);
REGISTER_OP_CPU_KERNEL(
    dot_grad, ops::DotGradKernel<paddle::platform::CPUDeviceContext, float>,
    ops::DotGradKernel<paddle::platform::CPUDeviceContext, double>,
    ops::DotGradKernel<paddle::platform::CPUDeviceContext, int>,
    ops::DotGradKernel<paddle::platform::CPUDeviceContext, int64_t>,
    ops::DotGradKernel<paddle::platform::CPUDeviceContext,
                       paddle::platform::complex<float>>,
    ops::DotGradKernel<paddle::platform::CPUDeviceContext,
                       paddle::platform::complex<double>>);
<commit_msg>remove dot infershape (#39945)<commit_after>// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "paddle/fluid/operators/dot_op.h"

#include "paddle/fluid/framework/infershape_utils.h"
#include "paddle/phi/core/infermeta_utils.h"
#include "paddle/phi/infermeta/binary.h"

namespace paddle {
namespace operators {

class DotOp : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

  framework::OpKernelType GetExpectedKernelType(
      const framework::ExecutionContext& ctx) const override {
    return framework::OpKernelType(
        OperatorWithKernel::IndicateVarDataType(ctx, "X"), ctx.GetPlace());
  }
};

class DotOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
  void Make() final {
    AddInput("X", "(Tensor) The first input tensor. ");
    AddInput("Y", "(Tensor) The second input tensor. ");
    AddOutput("Out", "(Tensor) The result tensor.");
    AddComment("");
  }
};

class DotGradOp : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

  void InferShape(framework::InferShapeContext* ctx) const override {
    PADDLE_ENFORCE_EQ(
        true, ctx->HasInput("X"),
        platform::errors::PreconditionNotMet("Input(X) should not be null."));
    PADDLE_ENFORCE_EQ(
        true, ctx->HasInput("Y"),
        platform::errors::PreconditionNotMet("Input(Y) should not be null."));
    PADDLE_ENFORCE_EQ(true, ctx->HasInput(framework::GradVarName("Out")),
                      platform::errors::PreconditionNotMet(
                          "Input(Out@GRAD) should not be null."));

    auto x_grad_name = framework::GradVarName("X");
    auto y_grad_name = framework::GradVarName("Y");
    if (ctx->HasOutput(x_grad_name)) {
      ctx->ShareDim("X", /*->*/ x_grad_name);
      ctx->ShareLoD("X", /*->*/ x_grad_name);
    }
    if (ctx->HasOutput(y_grad_name)) {
      ctx->ShareDim("Y", /*->*/ y_grad_name);
      ctx->ShareLoD("Y", /*->*/ y_grad_name);
    }
  }

 protected:
  framework::OpKernelType GetExpectedKernelType(
      const framework::ExecutionContext& ctx) const override {
    return framework::OpKernelType(OperatorWithKernel::IndicateVarDataType(
                                       ctx, framework::GradVarName("Out")),
                                   ctx.GetPlace());
  }
};

template <typename T>
class DotOpGradMaker : public framework::SingleGradOpMaker<T> {
 public:
  using framework::SingleGradOpMaker<T>::SingleGradOpMaker;

 protected:
  void Apply(GradOpPtr<T> op) const override {
    op->SetType("dot_grad");

    op->SetInput("X", this->Input("X"));
    op->SetInput("Y", this->Input("Y"));
    op->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out"));
    op->SetAttrMap(this->Attrs());
    op->SetOutput(framework::GradVarName("X"), this->InputGrad("X"));
    op->SetOutput(framework::GradVarName("Y"), this->InputGrad("Y"));
  }
};

}  // namespace operators
}  // namespace paddle

namespace ops = paddle::operators;

DELCARE_INFER_SHAPE_FUNCTOR(dot, DotInferShapeFunctor,
                            PT_INFER_META(phi::DotInferMeta));

REGISTER_OPERATOR(dot, ops::DotOp, ops::DotOpMaker,
                  ops::DotOpGradMaker<paddle::framework::OpDesc>,
                  ops::DotOpGradMaker<paddle::imperative::OpBase>,
                  DotInferShapeFunctor);

REGISTER_OPERATOR(dot_grad, ops::DotGradOp);

REGISTER_OP_CPU_KERNEL(
    dot, ops::DotKernel<paddle::platform::CPUDeviceContext, float>,
    ops::DotKernel<paddle::platform::CPUDeviceContext, double>,
    ops::DotKernel<paddle::platform::CPUDeviceContext, int>,
    ops::DotKernel<paddle::platform::CPUDeviceContext, int64_t>,
    ops::DotKernel<paddle::platform::CPUDeviceContext,
                   paddle::platform::complex<float>>,
    ops::DotKernel<paddle::platform::CPUDeviceContext,
                   paddle::platform::complex<double>>);
REGISTER_OP_CPU_KERNEL(
    dot_grad, ops::DotGradKernel<paddle::platform::CPUDeviceContext, float>,
    ops::DotGradKernel<paddle::platform::CPUDeviceContext, double>,
    ops::DotGradKernel<paddle::platform::CPUDeviceContext, int>,
    ops::DotGradKernel<paddle::platform::CPUDeviceContext, int64_t>,
    ops::DotGradKernel<paddle::platform::CPUDeviceContext,
                       paddle::platform::complex<float>>,
    ops::DotGradKernel<paddle::platform::CPUDeviceContext,
                       paddle::platform::complex<double>>);
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include <cstdint>
#include <cstdlib>
#include <ostream>
#include <memory>
#include <string>
#include <vector>

#include <gflags/gflags.h>
#include <gflags/gflags_declare.h>
#include <glog/logging.h>
#include <gtest/gtest.h>

#include "kudu/client/callbacks.h"
#include "kudu/client/client.h"
#include "kudu/client/row_result.h"
#include "kudu/client/schema.h"
#include "kudu/client/shared_ptr.h"
#include "kudu/client/write_op.h"
#include "kudu/common/partial_row.h"
#include "kudu/gutil/gscoped_ptr.h"
#include "kudu/gutil/port.h"
#include "kudu/gutil/ref_counted.h"
#include "kudu/gutil/strings/strcat.h"
#include "kudu/master/mini_master.h"
#include "kudu/mini-cluster/internal_mini_cluster.h"
#include "kudu/tserver/mini_tablet_server.h"
#include "kudu/util/async_util.h"
#include "kudu/util/countdown_latch.h"
#include "kudu/util/curl_util.h"
#include "kudu/util/faststring.h"
#include "kudu/util/monotime.h"
#include "kudu/util/net/sockaddr.h"
#include "kudu/util/status.h"
#include "kudu/util/stopwatch.h"
#include "kudu/util/test_macros.h"
#include "kudu/util/test_util.h"
#include "kudu/util/thread.h"

DECLARE_int32(flush_threshold_mb);
DECLARE_int32(log_segment_size_mb);
DECLARE_int32(maintenance_manager_polling_interval_ms);
DEFINE_int32(mbs_for_flushes_and_rolls, 1, "How many MBs are needed to flush and roll");
DEFINE_int32(row_count, 2000, "How many rows will be used in this test for the base data");
DEFINE_int32(seconds_to_run, 4,
             "How long this test runs for, after inserting the base data, in seconds");

using std::string;
using std::vector;

namespace kudu {
namespace tablet {

using client::KuduInsert;
using client::KuduClient;
using client::KuduClientBuilder;
using client::KuduColumnSchema;
using client::KuduRowResult;
using client::KuduScanner;
using client::KuduSchema;
using client::KuduSchemaBuilder;
using client::KuduSession;
using client::KuduStatusCallback;
using client::KuduStatusMemberCallback;
using client::KuduTable;
using client::KuduTableCreator;
using client::KuduUpdate;
using client::sp::shared_ptr;
using cluster::InternalMiniCluster;
using cluster::InternalMiniClusterOptions;

// This integration test tries to trigger all the update-related bits while also serving as a
// foundation for benchmarking. It first inserts 'row_count' rows and then starts two threads,
// one that continuously updates all the rows sequentially and one that scans them all, until
// it's been running for 'seconds_to_run'. It doesn't test for correctness, unless something
// FATALs.
class UpdateScanDeltaCompactionTest : public KuduTest {
 protected:
  UpdateScanDeltaCompactionTest() {
    KuduSchemaBuilder b;
    b.AddColumn("key")->Type(KuduColumnSchema::INT64)->NotNull()->PrimaryKey();
    b.AddColumn("string")->Type(KuduColumnSchema::STRING)->NotNull();
    b.AddColumn("int64")->Type(KuduColumnSchema::INT64)->NotNull();
    CHECK_OK(b.Build(&schema_));
  }

  virtual void SetUp() OVERRIDE {
    KuduTest::SetUp();
  }

  void CreateTable() {
    ASSERT_NO_FATAL_FAILURE(InitCluster());
    gscoped_ptr<KuduTableCreator> table_creator(client_->NewTableCreator());
    ASSERT_OK(table_creator->table_name(kTableName)
             .schema(&schema_)
             .set_range_partition_columns({ "key" })
             .num_replicas(1)
             .Create());
    ASSERT_OK(client_->OpenTable(kTableName, &table_));
  }

  virtual void TearDown() OVERRIDE {
    if (cluster_) {
      cluster_->Shutdown();
    }
    KuduTest::TearDown();
  }

  // Inserts row_count rows sequentially.
  void InsertBaseData();

  // Starts the update and scan threads then stops them after seconds_to_run.
  void RunThreads();

 private:
  enum {
    kKeyCol,
    kStrCol,
    kInt64Col
  };
  static const char* const kTableName;

  void InitCluster() {
    // Start mini-cluster with 1 tserver.
    cluster_.reset(new InternalMiniCluster(env_, InternalMiniClusterOptions()));
    ASSERT_OK(cluster_->Start());
    KuduClientBuilder client_builder;
    client_builder.add_master_server_addr(
        cluster_->mini_master()->bound_rpc_addr_str());
    ASSERT_OK(client_builder.Build(&client_));
  }

  shared_ptr<KuduSession> CreateSession() {
    shared_ptr<KuduSession> session = client_->NewSession();
    session->SetTimeoutMillis(5000);
    CHECK_OK(session->SetFlushMode(KuduSession::MANUAL_FLUSH));
    return session;
  }

  // Continuously updates the existing data until 'stop_latch' drops to 0.
  void UpdateRows(CountDownLatch* stop_latch);

  // Continuously scans the data until 'stop_latch' drops to 0.
  void ScanRows(CountDownLatch* stop_latch) const;

  // Continuously fetch various web pages on the TS.
  void CurlWebPages(CountDownLatch* stop_latch) const;

  // Sets the passed values on the row.
  // TODO randomize the string column.
  void MakeRow(int64_t key, int64_t val, KuduPartialRow* row) const;

  // If 'key' is a multiple of kSessionBatchSize, it uses 'last_s' to wait for the previous batch
  // to finish and then flushes the current one.
  Status WaitForLastBatchAndFlush(int64_t key,
                                  Synchronizer* last_s,
                                  KuduStatusCallback* last_s_cb,
                                  shared_ptr<KuduSession> session);

  KuduSchema schema_;
  std::shared_ptr<InternalMiniCluster> cluster_;
  shared_ptr<KuduTable> table_;
  shared_ptr<KuduClient> client_;
};

const char* const UpdateScanDeltaCompactionTest::kTableName = "update-scan-delta-compact-tbl";
const int kSessionBatchSize = 1000;

TEST_F(UpdateScanDeltaCompactionTest, TestAll) {
  OverrideFlagForSlowTests("seconds_to_run", "100");
  OverrideFlagForSlowTests("row_count", "1000000");
  OverrideFlagForSlowTests("mbs_for_flushes_and_rolls", "8");
  // Setting this high enough that we see the effects of flushes and compactions.
  OverrideFlagForSlowTests("maintenance_manager_polling_interval_ms", "2000");
  FLAGS_flush_threshold_mb = FLAGS_mbs_for_flushes_and_rolls;
  FLAGS_log_segment_size_mb = FLAGS_mbs_for_flushes_and_rolls;
  if (!AllowSlowTests()) {
    // Make it run more often since it's not a long test.
    FLAGS_maintenance_manager_polling_interval_ms = 50;
  }

  ASSERT_NO_FATAL_FAILURE(CreateTable());
  ASSERT_NO_FATAL_FAILURE(InsertBaseData());
  ASSERT_NO_FATAL_FAILURE(RunThreads());
}

void UpdateScanDeltaCompactionTest::InsertBaseData() {
  shared_ptr<KuduSession> session = CreateSession();
  Synchronizer last_s;
  KuduStatusMemberCallback<Synchronizer> last_s_cb(&last_s,
                                                   &Synchronizer::StatusCB);
  last_s_cb.Run(Status::OK());

  LOG_TIMING(INFO, "Insert") {
    for (int64_t key = 0; key < FLAGS_row_count; key++) {
      gscoped_ptr<KuduInsert> insert(table_->NewInsert());
      MakeRow(key, 0, insert->mutable_row());
      ASSERT_OK(session->Apply(insert.release()));
      ASSERT_OK(WaitForLastBatchAndFlush(key, &last_s, &last_s_cb, session));
    }
    ASSERT_OK(WaitForLastBatchAndFlush(kSessionBatchSize, &last_s, &last_s_cb, session));
    ASSERT_OK(last_s.Wait());
  }
}

void UpdateScanDeltaCompactionTest::RunThreads() {
  vector<scoped_refptr<Thread> > threads;

  CountDownLatch stop_latch(1);

  {
    scoped_refptr<Thread> t;
    ASSERT_OK(Thread::Create(CURRENT_TEST_NAME(),
                             StrCat(CURRENT_TEST_CASE_NAME(), "-update"),
                             &UpdateScanDeltaCompactionTest::UpdateRows, this,
                             &stop_latch, &t));
    threads.push_back(t);
  }

  {
    scoped_refptr<Thread> t;
    ASSERT_OK(Thread::Create(CURRENT_TEST_NAME(),
                             StrCat(CURRENT_TEST_CASE_NAME(), "-scan"),
                             &UpdateScanDeltaCompactionTest::ScanRows, this,
                             &stop_latch, &t));
    threads.push_back(t);
  }

  {
    scoped_refptr<Thread> t;
    ASSERT_OK(Thread::Create(CURRENT_TEST_NAME(),
                             StrCat(CURRENT_TEST_CASE_NAME(), "-curl"),
                             &UpdateScanDeltaCompactionTest::CurlWebPages, this,
                             &stop_latch, &t));
    threads.push_back(t);
  }

  SleepFor(MonoDelta::FromSeconds(FLAGS_seconds_to_run * 1.0));
  stop_latch.CountDown();

  for (const scoped_refptr<Thread>& thread : threads) {
    ASSERT_OK(ThreadJoiner(thread.get())
              .warn_every_ms(500)
              .Join());
  }
}

void UpdateScanDeltaCompactionTest::UpdateRows(CountDownLatch* stop_latch) {
  shared_ptr<KuduSession> session = CreateSession();
  Synchronizer last_s;
  KuduStatusMemberCallback<Synchronizer> last_s_cb(&last_s,
                                                   &Synchronizer::StatusCB);

  for (int64_t iteration = 1; stop_latch->count() > 0; iteration++) {
    last_s_cb.Run(Status::OK());
    LOG_TIMING(INFO, "Update") {
      for (int64_t key = 0; key < FLAGS_row_count && stop_latch->count() > 0; key++) {
        gscoped_ptr<KuduUpdate> update(table_->NewUpdate());
        MakeRow(key, iteration, update->mutable_row());
        CHECK_OK(session->Apply(update.release()));
        CHECK_OK(WaitForLastBatchAndFlush(key, &last_s, &last_s_cb, session));
      }
      CHECK_OK(WaitForLastBatchAndFlush(kSessionBatchSize, &last_s, &last_s_cb, session));
      CHECK_OK(last_s.Wait());
    }
  }
}

void UpdateScanDeltaCompactionTest::ScanRows(CountDownLatch* stop_latch) const {
  while (stop_latch->count() > 0) {
    KuduScanner scanner(table_.get());
    // Sometimes use fault-tolerant scans, to get more coverage of the
    // MergeIterator code paths.
    if (rand() % 2 == 1) {
      CHECK_OK(scanner.SetFaultTolerant());
    }
    LOG_TIMING(INFO, "Scan") {
      CHECK_OK(scanner.Open());
      vector<KuduRowResult> rows;
      while (scanner.HasMoreRows()) {
        CHECK_OK(scanner.NextBatch(&rows));
      }
    }
  }
}

void UpdateScanDeltaCompactionTest::CurlWebPages(CountDownLatch* stop_latch) const {
  vector<string> urls;
  string base_url = cluster_->mini_tablet_server(0)->bound_http_addr().ToString();
  urls.push_back(base_url + "/scans");
  urls.push_back(base_url + "/transactions");

  EasyCurl curl;
  faststring dst;
  while (stop_latch->count() > 0) {
    for (const string& url : urls) {
      VLOG(1) << "Curling URL " << url;
      Status status = curl.FetchURL(url, &dst);
      if (status.ok()) {
        CHECK_GT(dst.length(), 0);
      }
    }
  }
}

void UpdateScanDeltaCompactionTest::MakeRow(int64_t key,
                                            int64_t val,
                                            KuduPartialRow* row) const {
  CHECK_OK(row->SetInt64(kKeyCol, key));
  CHECK_OK(row->SetStringCopy(kStrCol, "TODO random string"));
  CHECK_OK(row->SetInt64(kInt64Col, val));
}

Status UpdateScanDeltaCompactionTest::WaitForLastBatchAndFlush(int64_t key,
                                                               Synchronizer* last_s,
                                                               KuduStatusCallback* last_s_cb,
                                                               shared_ptr<KuduSession> session) {
  if (key % kSessionBatchSize == 0) {
    RETURN_NOT_OK(last_s->Wait());
    last_s->Reset();
    session->FlushAsync(last_s_cb);
  }
  return Status::OK();
}

} // namespace tablet
} // namespace kudu
<commit_msg>KUDU-2222. update_scan_delta_compact-test: increase write timeout<commit_after>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include <cstdint>
#include <cstdlib>
#include <ostream>
#include <memory>
#include <string>
#include <vector>

#include <gflags/gflags.h>
#include <gflags/gflags_declare.h>
#include <glog/logging.h>
#include <gtest/gtest.h>

#include "kudu/client/callbacks.h"
#include "kudu/client/client.h"
#include "kudu/client/row_result.h"
#include "kudu/client/schema.h"
#include "kudu/client/shared_ptr.h"
#include "kudu/client/write_op.h"
#include "kudu/common/partial_row.h"
#include "kudu/gutil/gscoped_ptr.h"
#include "kudu/gutil/port.h"
#include "kudu/gutil/ref_counted.h"
#include "kudu/gutil/strings/strcat.h"
#include "kudu/master/mini_master.h"
#include "kudu/mini-cluster/internal_mini_cluster.h"
#include "kudu/tserver/mini_tablet_server.h"
#include "kudu/util/async_util.h"
#include "kudu/util/countdown_latch.h"
#include "kudu/util/curl_util.h"
#include "kudu/util/faststring.h"
#include "kudu/util/monotime.h"
#include "kudu/util/net/sockaddr.h"
#include "kudu/util/status.h"
#include "kudu/util/stopwatch.h"
#include "kudu/util/test_macros.h"
#include "kudu/util/test_util.h"
#include "kudu/util/thread.h"

DECLARE_int32(flush_threshold_mb);
DECLARE_int32(log_segment_size_mb);
DECLARE_int32(maintenance_manager_polling_interval_ms);
DEFINE_int32(mbs_for_flushes_and_rolls, 1, "How many MBs are needed to flush and roll");
DEFINE_int32(row_count, 2000, "How many rows will be used in this test for the base data");
DEFINE_int32(seconds_to_run, 4,
             "How long this test runs for, after inserting the base data, in seconds");

using std::string;
using std::vector;

namespace kudu {
namespace tablet {

using client::KuduInsert;
using client::KuduClient;
using client::KuduClientBuilder;
using client::KuduColumnSchema;
using client::KuduRowResult;
using client::KuduScanner;
using client::KuduSchema;
using client::KuduSchemaBuilder;
using client::KuduSession;
using client::KuduStatusCallback;
using client::KuduStatusMemberCallback;
using client::KuduTable;
using client::KuduTableCreator;
using client::KuduUpdate;
using client::sp::shared_ptr;
using cluster::InternalMiniCluster;
using cluster::InternalMiniClusterOptions;

// This integration test tries to trigger all the update-related bits while also serving as a
// foundation for benchmarking. It first inserts 'row_count' rows and then starts two threads,
// one that continuously updates all the rows sequentially and one that scans them all, until
// it's been running for 'seconds_to_run'. It doesn't test for correctness, unless something
// FATALs.
class UpdateScanDeltaCompactionTest : public KuduTest {
 protected:
  UpdateScanDeltaCompactionTest() {
    KuduSchemaBuilder b;
    b.AddColumn("key")->Type(KuduColumnSchema::INT64)->NotNull()->PrimaryKey();
    b.AddColumn("string")->Type(KuduColumnSchema::STRING)->NotNull();
    b.AddColumn("int64")->Type(KuduColumnSchema::INT64)->NotNull();
    CHECK_OK(b.Build(&schema_));
  }

  virtual void SetUp() OVERRIDE {
    KuduTest::SetUp();
  }

  void CreateTable() {
    ASSERT_NO_FATAL_FAILURE(InitCluster());
    gscoped_ptr<KuduTableCreator> table_creator(client_->NewTableCreator());
    ASSERT_OK(table_creator->table_name(kTableName)
             .schema(&schema_)
             .set_range_partition_columns({ "key" })
             .num_replicas(1)
             .Create());
    ASSERT_OK(client_->OpenTable(kTableName, &table_));
  }

  virtual void TearDown() OVERRIDE {
    if (cluster_) {
      cluster_->Shutdown();
    }
    KuduTest::TearDown();
  }

  // Inserts row_count rows sequentially.
  void InsertBaseData();

  // Starts the update and scan threads then stops them after seconds_to_run.
  void RunThreads();

 private:
  enum {
    kKeyCol,
    kStrCol,
    kInt64Col
  };
  static const char* const kTableName;

  void InitCluster() {
    // Start mini-cluster with 1 tserver.
    cluster_.reset(new InternalMiniCluster(env_, InternalMiniClusterOptions()));
    ASSERT_OK(cluster_->Start());
    KuduClientBuilder client_builder;
    client_builder.add_master_server_addr(
        cluster_->mini_master()->bound_rpc_addr_str());
    ASSERT_OK(client_builder.Build(&client_));
  }

  shared_ptr<KuduSession> CreateSession() {
    shared_ptr<KuduSession> session = client_->NewSession();
    session->SetTimeoutMillis(30000);
    CHECK_OK(session->SetFlushMode(KuduSession::MANUAL_FLUSH));
    return session;
  }

  // Continuously updates the existing data until 'stop_latch' drops to 0.
  void UpdateRows(CountDownLatch* stop_latch);

  // Continuously scans the data until 'stop_latch' drops to 0.
  void ScanRows(CountDownLatch* stop_latch) const;

  // Continuously fetch various web pages on the TS.
  void CurlWebPages(CountDownLatch* stop_latch) const;

  // Sets the passed values on the row.
  // TODO randomize the string column.
  void MakeRow(int64_t key, int64_t val, KuduPartialRow* row) const;

  // If 'key' is a multiple of kSessionBatchSize, it uses 'last_s' to wait for the previous batch
  // to finish and then flushes the current one.
  Status WaitForLastBatchAndFlush(int64_t key,
                                  Synchronizer* last_s,
                                  KuduStatusCallback* last_s_cb,
                                  shared_ptr<KuduSession> session);

  KuduSchema schema_;
  std::shared_ptr<InternalMiniCluster> cluster_;
  shared_ptr<KuduTable> table_;
  shared_ptr<KuduClient> client_;
};

const char* const UpdateScanDeltaCompactionTest::kTableName = "update-scan-delta-compact-tbl";
const int kSessionBatchSize = 1000;

TEST_F(UpdateScanDeltaCompactionTest, TestAll) {
  OverrideFlagForSlowTests("seconds_to_run", "100");
  OverrideFlagForSlowTests("row_count", "1000000");
  OverrideFlagForSlowTests("mbs_for_flushes_and_rolls", "8");
  // Setting this high enough that we see the effects of flushes and compactions.
  OverrideFlagForSlowTests("maintenance_manager_polling_interval_ms", "2000");
  FLAGS_flush_threshold_mb = FLAGS_mbs_for_flushes_and_rolls;
  FLAGS_log_segment_size_mb = FLAGS_mbs_for_flushes_and_rolls;
  if (!AllowSlowTests()) {
    // Make it run more often since it's not a long test.
    FLAGS_maintenance_manager_polling_interval_ms = 50;
  }

  ASSERT_NO_FATAL_FAILURE(CreateTable());
  ASSERT_NO_FATAL_FAILURE(InsertBaseData());
  ASSERT_NO_FATAL_FAILURE(RunThreads());
}

void UpdateScanDeltaCompactionTest::InsertBaseData() {
  shared_ptr<KuduSession> session = CreateSession();
  Synchronizer last_s;
  KuduStatusMemberCallback<Synchronizer> last_s_cb(&last_s,
                                                   &Synchronizer::StatusCB);
  last_s_cb.Run(Status::OK());

  LOG_TIMING(INFO, "Insert") {
    for (int64_t key = 0; key < FLAGS_row_count; key++) {
      gscoped_ptr<KuduInsert> insert(table_->NewInsert());
      MakeRow(key, 0, insert->mutable_row());
      ASSERT_OK(session->Apply(insert.release()));
      ASSERT_OK(WaitForLastBatchAndFlush(key, &last_s, &last_s_cb, session));
    }
    ASSERT_OK(WaitForLastBatchAndFlush(kSessionBatchSize, &last_s, &last_s_cb, session));
    ASSERT_OK(last_s.Wait());
  }
}

void UpdateScanDeltaCompactionTest::RunThreads() {
  vector<scoped_refptr<Thread> > threads;

  CountDownLatch stop_latch(1);

  {
    scoped_refptr<Thread> t;
    ASSERT_OK(Thread::Create(CURRENT_TEST_NAME(),
                             StrCat(CURRENT_TEST_CASE_NAME(), "-update"),
                             &UpdateScanDeltaCompactionTest::UpdateRows, this,
                             &stop_latch, &t));
    threads.push_back(t);
  }

  {
    scoped_refptr<Thread> t;
    ASSERT_OK(Thread::Create(CURRENT_TEST_NAME(),
                             StrCat(CURRENT_TEST_CASE_NAME(), "-scan"),
                             &UpdateScanDeltaCompactionTest::ScanRows, this,
                             &stop_latch, &t));
    threads.push_back(t);
  }

  {
    scoped_refptr<Thread> t;
    ASSERT_OK(Thread::Create(CURRENT_TEST_NAME(),
                             StrCat(CURRENT_TEST_CASE_NAME(), "-curl"),
                             &UpdateScanDeltaCompactionTest::CurlWebPages, this,
                             &stop_latch, &t));
    threads.push_back(t);
  }

  SleepFor(MonoDelta::FromSeconds(FLAGS_seconds_to_run * 1.0));
  stop_latch.CountDown();

  for (const scoped_refptr<Thread>& thread : threads) {
    ASSERT_OK(ThreadJoiner(thread.get())
              .warn_every_ms(500)
              .Join());
  }
}

void UpdateScanDeltaCompactionTest::UpdateRows(CountDownLatch* stop_latch) {
  shared_ptr<KuduSession> session = CreateSession();
  Synchronizer last_s;
  KuduStatusMemberCallback<Synchronizer> last_s_cb(&last_s,
                                                   &Synchronizer::StatusCB);

  for (int64_t iteration = 1; stop_latch->count() > 0; iteration++) {
    last_s_cb.Run(Status::OK());
    LOG_TIMING(INFO, "Update") {
      for (int64_t key = 0; key < FLAGS_row_count && stop_latch->count() > 0; key++) {
        gscoped_ptr<KuduUpdate> update(table_->NewUpdate());
        MakeRow(key, iteration, update->mutable_row());
        CHECK_OK(session->Apply(update.release()));
        CHECK_OK(WaitForLastBatchAndFlush(key, &last_s, &last_s_cb, session));
      }
      CHECK_OK(WaitForLastBatchAndFlush(kSessionBatchSize, &last_s, &last_s_cb, session));
      CHECK_OK(last_s.Wait());
    }
  }
}

void UpdateScanDeltaCompactionTest::ScanRows(CountDownLatch* stop_latch) const {
  while (stop_latch->count() > 0) {
    KuduScanner scanner(table_.get());
    // Sometimes use fault-tolerant scans, to get more coverage of the
    // MergeIterator code paths.
    if (rand() % 2 == 1) {
      CHECK_OK(scanner.SetFaultTolerant());
    }
    LOG_TIMING(INFO, "Scan") {
      CHECK_OK(scanner.Open());
      vector<KuduRowResult> rows;
      while (scanner.HasMoreRows()) {
        CHECK_OK(scanner.NextBatch(&rows));
      }
    }
  }
}

void UpdateScanDeltaCompactionTest::CurlWebPages(CountDownLatch* stop_latch) const {
  vector<string> urls;
  string base_url = cluster_->mini_tablet_server(0)->bound_http_addr().ToString();
  urls.push_back(base_url + "/scans");
  urls.push_back(base_url + "/transactions");

  EasyCurl curl;
  faststring dst;
  while (stop_latch->count() > 0) {
    for (const string& url : urls) {
      VLOG(1) << "Curling URL " << url;
      Status status = curl.FetchURL(url, &dst);
      if (status.ok()) {
        CHECK_GT(dst.length(), 0);
      }
    }
  }
}

void UpdateScanDeltaCompactionTest::MakeRow(int64_t key,
                                            int64_t val,
                                            KuduPartialRow* row) const {
  CHECK_OK(row->SetInt64(kKeyCol, key));
  CHECK_OK(row->SetStringCopy(kStrCol, "TODO random string"));
  CHECK_OK(row->SetInt64(kInt64Col, val));
}

Status UpdateScanDeltaCompactionTest::WaitForLastBatchAndFlush(int64_t key,
                                                               Synchronizer* last_s,
                                                               KuduStatusCallback* last_s_cb,
                                                               shared_ptr<KuduSession> session) {
  if (key % kSessionBatchSize == 0) {
    RETURN_NOT_OK(last_s->Wait());
    last_s->Reset();
    session->FlushAsync(last_s_cb);
  }
  return Status::OK();
}

} // namespace tablet
} // namespace kudu
<|endoftext|>


<commit_before>/*  This file is part of the KDE libraries and the Kate part.
 *
 *  Copyright (C) 2008-2011 Erlend Hamberg <ehamberg@gmail.com>
 *  Copyright (C) 2011 Svyatoslav Kuzmich <svatoslav1@gmail.com>
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Library General Public
 *  License as published by the Free Software Foundation; either
 *  version 2 of the License, or (at your option) any later version.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Library General Public License for more details.
 *
 *  You should have received a copy of the GNU Library General Public License
 *  along with this library; see the file COPYING.LIB.  If not, write to
 *  the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 *  Boston, MA 02110-1301, USA.
 */

#include "kateviinsertmode.h"
#include "kateviinputmodemanager.h"
#include "kateview.h"
#include "kateviewinternal.h"
#include "kateconfig.h"
#include "katecompletionwidget.h"
#include "kateglobal.h"
#include "katevikeyparser.h"

using KTextEditor::Cursor;

KateViInsertMode::KateViInsertMode( KateViInputModeManager *viInputModeManager,
    KateView * view, KateViewInternal * viewInternal ) : KateViModeBase()
{
  m_view = view;
  m_viewInternal = viewInternal;
  m_viInputModeManager = viInputModeManager;

  m_blockInsert = None;
  m_eolPos = 0;
}

KateViInsertMode::~KateViInsertMode()
{
}

bool KateViInsertMode::commandInsertFromAbove()
{
  Cursor c( m_view->cursorPosition() );

  if ( c.line() <= 0 ) {
    return false;
  }

  QString line = doc()->line( c.line()-1 );
  int tabWidth = doc()->config()->tabWidth();
  QChar ch = getCharAtVirtualColumn( line, m_view->virtualCursorColumn(), tabWidth );

  if ( ch == QChar::Null ) {
    return false;
  }

  return doc()->insertText( c, ch  );
}

bool KateViInsertMode::commandInsertFromBelow()
{
  Cursor c( m_view->cursorPosition() );

  if ( c.line() >= doc()->lines()-1 ) {
    return false;
  }

  QString line = doc()->line( c.line()+1 );
  int tabWidth = doc()->config()->tabWidth();
  QChar ch = getCharAtVirtualColumn( line, m_view->virtualCursorColumn(), tabWidth );

  if ( ch == QChar::Null ) {
    return false;
  }

  return doc()->insertText( c, ch );
}

bool KateViInsertMode::commandDeleteWord()
{
    Cursor c1( m_view->cursorPosition() );
    Cursor c2;

    c2 = findPrevWordStart( c1.line(), c1.column() );

    if ( c2.line() != c1.line() ) {
        if ( c1.column() == 0 ) {
            c2.setColumn( doc()->line( c2.line() ).length() );
        } else {
            c2.setColumn( 0 );
            c2.setLine( c2.line()+1 );
        }
    }

    KateViRange r( c2.line(), c2.column(), c1.line(), c1.column(), ViMotion::ExclusiveMotion );

    return deleteRange( r, CharWise, false );
}

bool KateViInsertMode::commandDeleteCharBackward()
{
    kDebug( 13070 ) << "Char backward!\n";
    Cursor c( m_view->cursorPosition() );

    KateViRange r( c.line(), c.column()-getCount(), c.line(), c.column(), ViMotion::ExclusiveMotion );

    if (c.column() == 0) {
        if (c.line() == 0) {
            return true;
        } else {
            r.startColumn = doc()->line(c.line()-1).length();
            r.startLine--;
      }
    }

    return deleteRange( r, CharWise );
}

bool KateViInsertMode::commandNewLine()
{
    doc()->newLine( m_view );
    return true;
}

bool KateViInsertMode::commandIndent()
{
    Cursor c( m_view->cursorPosition() );
    doc()->indent( KTextEditor::Range( c.line(), 0, c.line(), 0), 1 );
    return true;
}

bool KateViInsertMode::commandUnindent()
{
    Cursor c( m_view->cursorPosition() );
    doc()->indent( KTextEditor::Range( c.line(), 0, c.line(), 0), -1 );
    return true;
}

bool KateViInsertMode::commandToFirstCharacterInFile()
{
  Cursor c;

  c.setLine( 0 );
  c.setColumn( 0 );

  updateCursor( c );

  return true;
}

bool KateViInsertMode::commandToLastCharacterInFile()
{
  Cursor c;

  int lines = doc()->lines()-1;
  c.setLine( lines );
  c.setColumn( doc()->line( lines ).length() );

  updateCursor( c );

  return true;
}

bool KateViInsertMode::commandMoveOneWordLeft()
{
  Cursor c( m_view->cursorPosition() );
  c = findPrevWordStart( c.line(), c.column() );

  updateCursor( c );
  return true;
}

bool KateViInsertMode::commandMoveOneWordRight()
{
  Cursor c( m_view->cursorPosition() );
  c = findNextWordStart( c.line(), c.column() );

  updateCursor( c );
  return true;
}

bool KateViInsertMode::commandCompleteNext()
{
  if(m_view->completionWidget()->isCompletionActive()) {
    m_view->completionWidget()->cursorDown();
  } else {
    m_view->userInvokedCompletion();
  }
  return true;
}

bool KateViInsertMode::commandCompletePrevious()
{
  if(m_view->completionWidget()->isCompletionActive()) {
    m_view->completionWidget()->cursorUp();
  } else {
    m_view->userInvokedCompletion();
    m_view->completionWidget()->bottom();
  }
  return true;
}

bool KateViInsertMode::commandInsertContentOfRegister(){
    Cursor c( m_view->cursorPosition() );
    Cursor cAfter = c;
    QChar reg = getChosenRegister( m_register );

    OperationMode m = getRegisterFlag( reg );
    QString textToInsert = getRegisterContent( reg );

    if ( textToInsert.isNull() ) {
      error(i18n("Nothing in register %1", reg ));
      return false;
    }

    if ( m == LineWise ) {
      textToInsert.chop( 1 ); // remove the last \n
      c.setColumn( doc()->lineLength( c.line() ) ); // paste after the current line and ...
      textToInsert.prepend( QChar( '\n' ) ); // ... prepend a \n, so the text starts on a new line

      cAfter.setLine( cAfter.line()+1 );
      cAfter.setColumn( 0 );
    }

    doc()->insertText( c, textToInsert, m == Block );

    updateCursor( cAfter );

    return true;
}

// Start Normal mode just for one command and return to Insert mode
bool KateViInsertMode::commandSwitchToNormalModeForJustOneCommand(){
    m_viInputModeManager->setTemporaryNormalMode(true);
    m_viInputModeManager->changeViMode(NormalMode);
    m_view->setCaretStyle( KateRenderer::Block, true );
    m_view->updateViModeBarMode();
    m_viewInternal->repaint();
    return true;
}

/**
 * checks if the key is a valid command
 * @return true if a command was completed and executed, false otherwise
 */
bool KateViInsertMode::handleKeypress( const QKeyEvent *e )
{
  // backspace should work even if the shift key is down
  if (e->modifiers() != Qt::ControlModifier && e->key() == Qt::Key_Backspace ) {
    m_view->backspace();
    return true;
  }

  if(m_keys.isEmpty()){
  if ( e->modifiers() == Qt::NoModifier ) {
    switch ( e->key() ) {
    case Qt::Key_Escape:
      leaveInsertMode();
      return true;
      break;
    case Qt::Key_Left:
      m_view->cursorLeft();
      return true;
    case Qt::Key_Right:
      m_view->cursorRight();
      return true;
    case Qt::Key_Up:
      m_view->up();
      return true;
    case Qt::Key_Down:
      m_view->down();
      return true;
    case Qt::Key_Delete:
      m_view->keyDelete();
      return true;
    case Qt::Key_Home:
      m_view->home();
      return true;
    case Qt::Key_End:
      m_view->end();
      return true;
    case Qt::Key_PageUp:
      m_view->pageUp();
      return true;
    case Qt::Key_PageDown:
      m_view->pageDown();
      return true;
    default:
      return false;
      break;
    }
  } else if ( e->modifiers() == Qt::ControlModifier ) {
    switch( e->key() ) {
    case Qt::Key_BracketLeft:
    case Qt::Key_3:
      leaveInsertMode();
      return true;
      break;
    case Qt::Key_C:
      leaveInsertMode( true );
      return true;
      break;
    case Qt::Key_D:
      commandUnindent();
      return true;
      break;
    case Qt::Key_E:
      commandInsertFromBelow();
      return true;
      break;
    case Qt::Key_N:
      commandCompleteNext();
      return true;
      break;
    case Qt::Key_P:
      commandCompletePrevious();
      return true;
      break;
    case Qt::Key_T:
      commandIndent();
      return true;
      break;
    case Qt::Key_W:
      commandDeleteWord();
      return true;
      break;
    case Qt::Key_J:
      commandNewLine();
      return true;
      break;
    case Qt::Key_H:
      commandDeleteCharBackward();
      return true;
      break;
    case Qt::Key_Y:
      commandInsertFromAbove();
      return true;
      break;
    case Qt::Key_O:
      commandSwitchToNormalModeForJustOneCommand();
      return true;
      break;
    case Qt::Key_Home:
      commandToFirstCharacterInFile();
      return true;
      break;
    case Qt::Key_R:
      m_keys = "cR";
      // Waiting for register
      return true;
      break;
    case Qt::Key_End:
      commandToLastCharacterInFile();
      return true;
      break;
    case Qt::Key_Left:
      commandMoveOneWordLeft();
      return true;
      break;
    case Qt::Key_Right:
      commandMoveOneWordRight();
      return true;
      break;
    default:
      return false;
    }
  }

  return false;
} else {

    // Was waiting for register for Ctrl-R
    if (m_keys == "cR"){
        QChar key = KateViKeyParser::self()->KeyEventToQChar(
                    e->key(),
                    e->text(),
                    e->modifiers(),
                    e->nativeScanCode() );
        key = key.toLower();

        // is it register ?
        if ( ( key >= '0' && key <= '9' ) || ( key >= 'a' && key <= 'z' ) ||
                key == '_' || key == '+' || key == '*' ) {
          m_register = key;
        } else {
          m_keys = "";
          return false;
        }
        commandInsertContentOfRegister();
        m_keys = "";
        return true;
    }
  }
  return false;
}

// leave insert mode when esc, etc, is pressed. if leaving block
// prepend/append, the inserted text will be added to all block lines. if
// ctrl-c is used to exit insert mode this is not done.
void KateViInsertMode::leaveInsertMode( bool force )
{
    if ( !force && m_blockInsert != None ) { // block append/prepend

        // make sure cursor haven't been moved
        if ( m_blockRange.startLine == m_view->cursorPosition().line() ) {
            int start, len;
            QString added;
            Cursor c;

            switch ( m_blockInsert ) {
            case Append:
            case Prepend:
                if ( m_blockInsert == Append ) {
                    start = m_blockRange.endColumn+1;
                } else {
                    start = m_blockRange.startColumn;
                }

                len = m_view->cursorPosition().column()-start;
                added = getLine().mid( start, len );

                c = Cursor( m_blockRange.startLine, start );
                for ( int i = m_blockRange.startLine+1; i <= m_blockRange.endLine; i++ ) {
                    c.setLine( i );
                    doc()->insertText( c, added );
                }
                break;
            case AppendEOL:
                start = m_eolPos;
                len = m_view->cursorPosition().column()-start;
                added = getLine().mid( start, len );

                c = Cursor( m_blockRange.startLine, start );
                for ( int i = m_blockRange.startLine+1; i <= m_blockRange.endLine; i++ ) {
                    c.setLine( i );
                    c.setColumn( doc()->lineLength( i ) );
                    doc()->insertText( c, added );
                }
                break;
            default:
                error("not supported");
            }
        }

        m_blockInsert = None;
    }
    startNormalMode();
}

void KateViInsertMode::setBlockPrependMode( KateViRange blockRange )
{
    // ignore if not more than one line is selected
    if ( blockRange.startLine != blockRange.endLine ) {
        m_blockInsert = Prepend;
        m_blockRange = blockRange;
    }
}

void KateViInsertMode::setBlockAppendMode( KateViRange blockRange, BlockInsert b )
{
    Q_ASSERT( b == Append || b == AppendEOL );

    if ( b == AppendEOL ) {
        m_eolPos = doc()->lineLength( m_blockRange.startLine );
    }
    // ignore if not more than one line is selected
    if ( blockRange.startLine != blockRange.endLine ) {
        m_blockRange = blockRange;
        m_blockInsert = b;
    } else {
        kDebug( 13070 ) << "cursor moved. ignoring block append/prepend";
    }
}
<commit_msg>Enable insert repetition when completing text<commit_after>/*  This file is part of the KDE libraries and the Kate part.
 *
 *  Copyright (C) 2008-2011 Erlend Hamberg <ehamberg@gmail.com>
 *  Copyright (C) 2011 Svyatoslav Kuzmich <svatoslav1@gmail.com>
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Library General Public
 *  License as published by the Free Software Foundation; either
 *  version 2 of the License, or (at your option) any later version.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Library General Public License for more details.
 *
 *  You should have received a copy of the GNU Library General Public License
 *  along with this library; see the file COPYING.LIB.  If not, write to
 *  the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 *  Boston, MA 02110-1301, USA.
 */

#include "kateviinsertmode.h"
#include "kateviinputmodemanager.h"
#include "kateview.h"
#include "kateviewinternal.h"
#include "kateconfig.h"
#include "katecompletionwidget.h"
#include "kateglobal.h"
#include "katevikeyparser.h"

using KTextEditor::Cursor;

KateViInsertMode::KateViInsertMode( KateViInputModeManager *viInputModeManager,
    KateView * view, KateViewInternal * viewInternal ) : KateViModeBase()
{
  m_view = view;
  m_viewInternal = viewInternal;
  m_viInputModeManager = viInputModeManager;

  m_blockInsert = None;
  m_eolPos = 0;
}

KateViInsertMode::~KateViInsertMode()
{
}

bool KateViInsertMode::commandInsertFromAbove()
{
  Cursor c( m_view->cursorPosition() );

  if ( c.line() <= 0 ) {
    return false;
  }

  QString line = doc()->line( c.line()-1 );
  int tabWidth = doc()->config()->tabWidth();
  QChar ch = getCharAtVirtualColumn( line, m_view->virtualCursorColumn(), tabWidth );

  if ( ch == QChar::Null ) {
    return false;
  }

  return doc()->insertText( c, ch  );
}

bool KateViInsertMode::commandInsertFromBelow()
{
  Cursor c( m_view->cursorPosition() );

  if ( c.line() >= doc()->lines()-1 ) {
    return false;
  }

  QString line = doc()->line( c.line()+1 );
  int tabWidth = doc()->config()->tabWidth();
  QChar ch = getCharAtVirtualColumn( line, m_view->virtualCursorColumn(), tabWidth );

  if ( ch == QChar::Null ) {
    return false;
  }

  return doc()->insertText( c, ch );
}

bool KateViInsertMode::commandDeleteWord()
{
    Cursor c1( m_view->cursorPosition() );
    Cursor c2;

    c2 = findPrevWordStart( c1.line(), c1.column() );

    if ( c2.line() != c1.line() ) {
        if ( c1.column() == 0 ) {
            c2.setColumn( doc()->line( c2.line() ).length() );
        } else {
            c2.setColumn( 0 );
            c2.setLine( c2.line()+1 );
        }
    }

    KateViRange r( c2.line(), c2.column(), c1.line(), c1.column(), ViMotion::ExclusiveMotion );

    return deleteRange( r, CharWise, false );
}

bool KateViInsertMode::commandDeleteCharBackward()
{
    kDebug( 13070 ) << "Char backward!\n";
    Cursor c( m_view->cursorPosition() );

    KateViRange r( c.line(), c.column()-getCount(), c.line(), c.column(), ViMotion::ExclusiveMotion );

    if (c.column() == 0) {
        if (c.line() == 0) {
            return true;
        } else {
            r.startColumn = doc()->line(c.line()-1).length();
            r.startLine--;
      }
    }

    return deleteRange( r, CharWise );
}

bool KateViInsertMode::commandNewLine()
{
    doc()->newLine( m_view );
    return true;
}

bool KateViInsertMode::commandIndent()
{
    Cursor c( m_view->cursorPosition() );
    doc()->indent( KTextEditor::Range( c.line(), 0, c.line(), 0), 1 );
    return true;
}

bool KateViInsertMode::commandUnindent()
{
    Cursor c( m_view->cursorPosition() );
    doc()->indent( KTextEditor::Range( c.line(), 0, c.line(), 0), -1 );
    return true;
}

bool KateViInsertMode::commandToFirstCharacterInFile()
{
  Cursor c;

  c.setLine( 0 );
  c.setColumn( 0 );

  updateCursor( c );

  return true;
}

bool KateViInsertMode::commandToLastCharacterInFile()
{
  Cursor c;

  int lines = doc()->lines()-1;
  c.setLine( lines );
  c.setColumn( doc()->line( lines ).length() );

  updateCursor( c );

  return true;
}

bool KateViInsertMode::commandMoveOneWordLeft()
{
  Cursor c( m_view->cursorPosition() );
  c = findPrevWordStart( c.line(), c.column() );

  updateCursor( c );
  return true;
}

bool KateViInsertMode::commandMoveOneWordRight()
{
  Cursor c( m_view->cursorPosition() );
  c = findNextWordStart( c.line(), c.column() );

  updateCursor( c );
  return true;
}

bool KateViInsertMode::commandCompleteNext()
{
  // Completion may yield different results when performed
  // other places in the document. Therefore, repeat the
  // inserted text instead of the typed keystrokes.
  m_viInputModeManager->setTextualRepeat();
  if(m_view->completionWidget()->isCompletionActive()) {
    m_view->completionWidget()->cursorDown();
  } else {
    m_view->userInvokedCompletion();
  }
  return true;
}

bool KateViInsertMode::commandCompletePrevious()
{
  m_viInputModeManager->setTextualRepeat();
  if(m_view->completionWidget()->isCompletionActive()) {
    m_view->completionWidget()->cursorUp();
  } else {
    m_view->userInvokedCompletion();
    m_view->completionWidget()->bottom();
  }
  return true;
}

bool KateViInsertMode::commandInsertContentOfRegister(){
    Cursor c( m_view->cursorPosition() );
    Cursor cAfter = c;
    QChar reg = getChosenRegister( m_register );

    OperationMode m = getRegisterFlag( reg );
    QString textToInsert = getRegisterContent( reg );

    if ( textToInsert.isNull() ) {
      error(i18n("Nothing in register %1", reg ));
      return false;
    }

    if ( m == LineWise ) {
      textToInsert.chop( 1 ); // remove the last \n
      c.setColumn( doc()->lineLength( c.line() ) ); // paste after the current line and ...
      textToInsert.prepend( QChar( '\n' ) ); // ... prepend a \n, so the text starts on a new line

      cAfter.setLine( cAfter.line()+1 );
      cAfter.setColumn( 0 );
    }

    doc()->insertText( c, textToInsert, m == Block );

    updateCursor( cAfter );

    return true;
}

// Start Normal mode just for one command and return to Insert mode
bool KateViInsertMode::commandSwitchToNormalModeForJustOneCommand(){
    m_viInputModeManager->setTemporaryNormalMode(true);
    m_viInputModeManager->changeViMode(NormalMode);
    m_view->setCaretStyle( KateRenderer::Block, true );
    m_view->updateViModeBarMode();
    m_viewInternal->repaint();
    return true;
}

/**
 * checks if the key is a valid command
 * @return true if a command was completed and executed, false otherwise
 */
bool KateViInsertMode::handleKeypress( const QKeyEvent *e )
{
  // backspace should work even if the shift key is down
  if (e->modifiers() != Qt::ControlModifier && e->key() == Qt::Key_Backspace ) {
    m_view->backspace();
    return true;
  }

  if(m_keys.isEmpty()){
  if ( e->modifiers() == Qt::NoModifier ) {
    switch ( e->key() ) {
    case Qt::Key_Escape:
      leaveInsertMode();
      return true;
      break;
    case Qt::Key_Left:
      m_view->cursorLeft();
      return true;
    case Qt::Key_Right:
      m_view->cursorRight();
      return true;
    case Qt::Key_Up:
      m_view->up();
      return true;
    case Qt::Key_Down:
      m_view->down();
      return true;
    case Qt::Key_Delete:
      m_view->keyDelete();
      return true;
    case Qt::Key_Home:
      m_view->home();
      return true;
    case Qt::Key_End:
      m_view->end();
      return true;
    case Qt::Key_PageUp:
      m_view->pageUp();
      return true;
    case Qt::Key_PageDown:
      m_view->pageDown();
      return true;
    default:
      return false;
      break;
    }
  } else if ( e->modifiers() == Qt::ControlModifier ) {
    switch( e->key() ) {
    case Qt::Key_BracketLeft:
    case Qt::Key_3:
      leaveInsertMode();
      return true;
      break;
    case Qt::Key_Space:
      commandCompleteNext();
      return true;
      break;
    case Qt::Key_C:
      leaveInsertMode( true );
      return true;
      break;
    case Qt::Key_D:
      commandUnindent();
      return true;
      break;
    case Qt::Key_E:
      commandInsertFromBelow();
      return true;
      break;
    case Qt::Key_N:
      commandCompleteNext();
      return true;
      break;
    case Qt::Key_P:
      commandCompletePrevious();
      return true;
      break;
    case Qt::Key_T:
      commandIndent();
      return true;
      break;
    case Qt::Key_W:
      commandDeleteWord();
      return true;
      break;
    case Qt::Key_J:
      commandNewLine();
      return true;
      break;
    case Qt::Key_H:
      commandDeleteCharBackward();
      return true;
      break;
    case Qt::Key_Y:
      commandInsertFromAbove();
      return true;
      break;
    case Qt::Key_O:
      commandSwitchToNormalModeForJustOneCommand();
      return true;
      break;
    case Qt::Key_Home:
      commandToFirstCharacterInFile();
      return true;
      break;
    case Qt::Key_R:
      m_keys = "cR";
      // Waiting for register
      return true;
      break;
    case Qt::Key_End:
      commandToLastCharacterInFile();
      return true;
      break;
    case Qt::Key_Left:
      commandMoveOneWordLeft();
      return true;
      break;
    case Qt::Key_Right:
      commandMoveOneWordRight();
      return true;
      break;
    default:
      return false;
    }
  }

  return false;
} else {

    // Was waiting for register for Ctrl-R
    if (m_keys == "cR"){
        QChar key = KateViKeyParser::self()->KeyEventToQChar(
                    e->key(),
                    e->text(),
                    e->modifiers(),
                    e->nativeScanCode() );
        key = key.toLower();

        // is it register ?
        if ( ( key >= '0' && key <= '9' ) || ( key >= 'a' && key <= 'z' ) ||
                key == '_' || key == '+' || key == '*' ) {
          m_register = key;
        } else {
          m_keys = "";
          return false;
        }
        commandInsertContentOfRegister();
        m_keys = "";
        return true;
    }
  }
  return false;
}

// leave insert mode when esc, etc, is pressed. if leaving block
// prepend/append, the inserted text will be added to all block lines. if
// ctrl-c is used to exit insert mode this is not done.
void KateViInsertMode::leaveInsertMode( bool force )
{
    if ( !force && m_blockInsert != None ) { // block append/prepend

        // make sure cursor haven't been moved
        if ( m_blockRange.startLine == m_view->cursorPosition().line() ) {
            int start, len;
            QString added;
            Cursor c;

            switch ( m_blockInsert ) {
            case Append:
            case Prepend:
                if ( m_blockInsert == Append ) {
                    start = m_blockRange.endColumn+1;
                } else {
                    start = m_blockRange.startColumn;
                }

                len = m_view->cursorPosition().column()-start;
                added = getLine().mid( start, len );

                c = Cursor( m_blockRange.startLine, start );
                for ( int i = m_blockRange.startLine+1; i <= m_blockRange.endLine; i++ ) {
                    c.setLine( i );
                    doc()->insertText( c, added );
                }
                break;
            case AppendEOL:
                start = m_eolPos;
                len = m_view->cursorPosition().column()-start;
                added = getLine().mid( start, len );

                c = Cursor( m_blockRange.startLine, start );
                for ( int i = m_blockRange.startLine+1; i <= m_blockRange.endLine; i++ ) {
                    c.setLine( i );
                    c.setColumn( doc()->lineLength( i ) );
                    doc()->insertText( c, added );
                }
                break;
            default:
                error("not supported");
            }
        }

        m_blockInsert = None;
    }
    startNormalMode();
}

void KateViInsertMode::setBlockPrependMode( KateViRange blockRange )
{
    // ignore if not more than one line is selected
    if ( blockRange.startLine != blockRange.endLine ) {
        m_blockInsert = Prepend;
        m_blockRange = blockRange;
    }
}

void KateViInsertMode::setBlockAppendMode( KateViRange blockRange, BlockInsert b )
{
    Q_ASSERT( b == Append || b == AppendEOL );

    if ( b == AppendEOL ) {
        m_eolPos = doc()->lineLength( m_blockRange.startLine );
    }
    // ignore if not more than one line is selected
    if ( blockRange.startLine != blockRange.endLine ) {
        m_blockRange = blockRange;
        m_blockInsert = b;
    } else {
        kDebug( 13070 ) << "cursor moved. ignoring block append/prepend";
    }
}
<|endoftext|>


<commit_before>/**
 * Copyright (c) 2016 DeepCortex GmbH <legal@eventql.io>
 * Authors:
 *   - Paul Asmuth <paul@eventql.io>
 *   - Laura Schlimmer <laura@eventql.io>
 *
 * This program is free software: you can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License ("the license") as
 * published by the Free Software Foundation, either version 3 of the License,
 * or any later version.
 *
 * In accordance with Section 7(e) of the license, the licensing of the Program
 * under the license does not imply a trademark license. Therefore any rights,
 * title and interest in our trademarks remain entirely with us.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the license for more details.
 *
 * You can be released from the requirements of the license by purchasing a
 * commercial license. Buying such a license is mandatory as soon as you develop
 * commercial activities involving this program without disclosing the source
 * code of your own applications
 */
#include "eventql/sql/svalue.h"
#include "eventql/server/sql/partition_cursor.h"
#include "eventql/transport/native/frames/error.h"
#include "eventql/transport/native/frames/query_remote.h"
#include "eventql/transport/native/frames/query_remote_result.h"
#include "eventql/db/database.h"

namespace eventql {

PartitionCursor::PartitionCursor(
    csql::Transaction* txn,
    csql::ExecutionContext* execution_context,
    RefPtr<Table> table,
    RefPtr<PartitionSnapshot> snap,
    RefPtr<csql::SequentialScanNode> stmt) :
    txn_(txn),
    execution_context_(execution_context),
    table_(table),
    snap_(snap),
    stmt_(stmt),
    cur_table_(0),
    fast_scan_enabled_(true) {
  if (stmt_->aggregationStrategy() != csql::AggregationStrategy::NO_AGGREGATION) {
    fast_scan_enabled_ = false;
  }
}

ReturnCode PartitionCursor::execute() {
  return ReturnCode::success();
}

ReturnCode PartitionCursor::nextBatch(
    csql::SVector* columns,
    size_t* nrows) {
  for (;;) {
    if (cur_scan_.get() == nullptr) {
      if (!openNextTable()) {
        break;
      }
    }

    auto rc = cur_scan_->nextBatch(columns, nrows);
    if (!rc.isSuccess()) {
      return rc;
    }

    if (!*nrows) {
      cur_scan_.reset(nullptr);
      continue;
    }

    return ReturnCode::success();
  }

  *nrows = 0;
  return ReturnCode::success();
}

bool PartitionCursor::openNextTable() {
  RefPtr<cstable::CSTableReader> cstable;
  String cstable_filename;
  RefPtr<cstable::ColumnReader> skip_col;

  cur_skiplist_.reset();

  switch (cur_table_) {
    case 0: {
      if (snap_->head_arena.get() &&
          snap_->head_arena->getCSTableFile()) {
        cur_skiplist_.reset(
            new PartitionArena::SkiplistReader(
                snap_->head_arena->getSkiplistReader()));
        cstable = cstable::CSTableReader::openFile(
            snap_->head_arena->getCSTableFile(),
            cur_skiplist_->size());
        cstable_filename = StringUtil::format(
            "memory://$0/$1/$2/head_arena",
            snap_->state.tsdb_namespace(),
            snap_->state.table_key(),
            snap_->key.toString());
        break;
      } else {
        ++cur_table_;
        /* fallthrough */
      }
    }

    case 1: {
      if (snap_->compacting_arena.get() &&
          snap_->compacting_arena->getCSTableFile()) {
        cur_skiplist_.reset(
            new PartitionArena::SkiplistReader(
                snap_->compacting_arena->getSkiplistReader()));
        cstable = cstable::CSTableReader::openFile(
            snap_->compacting_arena->getCSTableFile(),
            cur_skiplist_->size());
        cstable_filename = StringUtil::format(
            "memory://$0/$1/$2/compacting_arena",
            snap_->state.tsdb_namespace(),
            snap_->state.table_key(),
            snap_->key.toString());
        break;
      } else {
        ++cur_table_;
        /* fallthrough */
      }
    }

    default: {
      if (cur_table_ >= snap_->state.lsm_tables().size() + 2) {
        return false;
      }
      const auto& tbl = (snap_->state.lsm_tables().data())[
          snap_->state.lsm_tables().size() - (cur_table_ - 1)];
      auto cstable_file = FileUtil::joinPaths(
          snap_->base_path,
          tbl->filename() + ".cst");
      cstable = cstable::CSTableReader::openFile(cstable_file);
      cstable_filename = cstable_file;
      if (tbl->has_skiplist()) {
        skip_col = cstable->getColumnReader("__lsm_skip");
      }
      break;
    }
  }

  /* read filter */
  std::vector<bool> filter(cstable->numRecords(), true);
  auto id_col = cstable->getColumnReader("__lsm_id");
  auto is_update_col = cstable->getColumnReader("__lsm_is_update");
  for (size_t i = 0; i < filter.size(); ++i) {
    uint64_t rlvl;
    uint64_t dlvl;

    String id_str;
    id_col->readString(&rlvl, &dlvl, &id_str);
    SHA1Hash id(id_str.data(), id_str.size());

    bool is_update;
    is_update_col->readBoolean(&rlvl, &dlvl, &is_update);

    bool skip = false;
    if (skip_col.get()) {
      skip_col->readBoolean(&rlvl, &dlvl, &skip);
    }

    if (cur_skiplist_) {
      skip = cur_skiplist_->readNext();
    }

    if (skip || id_set_.count(id) > 0) {
      filter[i] = false;
    } else {
      if (is_update) {
        id_set_.emplace(id);
      }

      filter[i] = true;
    }
  }

  if (fast_scan_enabled_) {
    cur_scan_.reset(
        new csql::FastCSTableScan(
            txn_,
            execution_context_,
            stmt_,
            cstable,
            cstable_filename));
  } else {
    cur_scan_.reset(
        new csql::CSTableScan(
            txn_,
            execution_context_,
            stmt_,
            cstable,
            cstable_filename));
  }

  cur_scan_->setFilter(std::move(filter));

  ++cur_table_;

  auto rc = cur_scan_->execute();
  if (!rc.isSuccess()) {
    RAISE(kRuntimeError, rc.getMessage());
  }

  return true;
}

size_t PartitionCursor::getColumnCount() const {
  return stmt_->getNumComputedColumns();
}

csql::SType PartitionCursor::getColumnType(size_t idx) const {
  return stmt_->getColumnType(idx);
}

RemotePartitionCursor::RemotePartitionCursor(
    Session* session,
    csql::Transaction* txn,
    csql::ExecutionContext* execution_context,
    const std::string& database,
    RefPtr<csql::SequentialScanNode> stmt,
    const std::vector<std::string>& servers) :
    txn_(txn),
    execution_context_(execution_context),
    database_(database),
    stmt_(stmt),
    servers_(servers),
    ncols_(stmt->getNumComputedColumns()),
    row_buf_pos_(0),
    running_(false),
    done_(false),
    timeout_(
        session->getDatabaseContext()->config->getInt(
            "server.s2s_io_timeout").get()),
    client_(
        session->getDatabaseContext()->connection_pool,
        session->getDatabaseContext()->dns_cache,
        session->getDatabaseContext()->config->getInt(
            "server.s2s_io_timeout",
            0),
        session->getDatabaseContext()->config->getInt(
            "server.s2s_idle_timeout",
            0)) {}

ReturnCode RemotePartitionCursor::execute() {
  return ReturnCode::success();
}

ReturnCode RemotePartitionCursor::nextBatch(
    csql::SVector* output,
    size_t* output_len) {
  if (done_) {
    *output_len = 0;
    return ReturnCode::success();
  }

  if (running_) {
    auto rc = client_.sendFrame(EVQL_OP_QUERY_CONTINUE, 0, nullptr, 0);
    if (!rc.isSuccess()) {
      return rc;
    }
  } else {
    auto session = static_cast<eventql::Session*>(txn_->getUserData());
    auto cdir = session->getDatabaseContext()->config_directory;

    std::string qtree_coded;
    auto qtree_coded_os = StringOutputStream::fromString(&qtree_coded);
    csql::QueryTreeCoder qtree_coder(txn_);
    qtree_coder.encode(stmt_.get(), qtree_coded_os.get());

    native_transport::QueryRemoteFrame q_frame;
    q_frame.setEncodedQtree(std::move(qtree_coded));
    q_frame.setDatabase(session->getEffectiveNamespace());

    for (const auto& s : servers_) {
      auto server_cfg = cdir->getServerConfig(s);
      if (server_cfg.server_status() != SERVER_UP) {
        logError("evqld", "server is down: $0", s);
        continue;
      }

      auto rc = client_.connect(server_cfg.server_addr(), true);
      if (!rc.isSuccess()) {
        logError("evqld", "Remote SQL Error: $0", rc.getMessage());
        continue;
      }

      rc = client_.sendFrame(&q_frame, 0);
      if (rc.isSuccess()) {
        running_ = true;
      } else {
        logError("evqld", "Remote SQL Error: $0", rc.getMessage());
        continue;
      }
    }

    if (!running_) {
      return ReturnCode::error("ERUNTIME", "no server available for query");
    }
  }

  uint16_t ret_opcode = 0;
  uint16_t ret_flags;
  std::string ret_payload;
  while (ret_opcode != EVQL_OP_QUERY_REMOTE_RESULT) {
    auto rc = txn_->triggerHeartbeat();
    if (!rc.isSuccess()) {
      return rc;
    }

    rc = client_.recvFrame(&ret_opcode, &ret_flags, &ret_payload, timeout_);
    if (!rc.isSuccess()) {
      return rc;
    }

    if (ret_flags & EVQL_ENDOFREQUEST) {
      done_ = true;
    }

    switch (ret_opcode) {
      case EVQL_OP_HEARTBEAT:
        continue;
      case EVQL_OP_QUERY_REMOTE_RESULT:
        break;
      case EVQL_OP_ERROR: {
        native_transport::ErrorFrame eframe;
        eframe.parseFrom(ret_payload.data(), ret_payload.size());
        return ReturnCode::error("ERUNTIME", eframe.getError());
      }
      default:
        return ReturnCode::error("ERUNTIME", "invalid opcode");
    }
  }

  native_transport::QueryRemoteResultFrame r_frame;
  auto rc = r_frame.parseFrom(ret_payload.data(), ret_payload.size());

  if (r_frame.getColumnCount() != ncols_) {
    return ReturnCode::error("ERUNTIME", "invalid column count");
  }

  for (size_t i = 0; i < ncols_; ++i) {
    const auto& coldata = r_frame.getColumnData(i);
    output[i].append(coldata.data(), coldata.size());
  }

  *output_len = r_frame.getRowCount();
  return ReturnCode::success();
}

size_t RemotePartitionCursor::getColumnCount() const {
  return stmt_->getNumComputedColumns();
}

csql::SType RemotePartitionCursor::getColumnType(size_t idx) const {
  return stmt_->getColumnType(idx);
}

}
<commit_msg>skip CSTable filter if possible<commit_after>/**
 * Copyright (c) 2016 DeepCortex GmbH <legal@eventql.io>
 * Authors:
 *   - Paul Asmuth <paul@eventql.io>
 *   - Laura Schlimmer <laura@eventql.io>
 *
 * This program is free software: you can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License ("the license") as
 * published by the Free Software Foundation, either version 3 of the License,
 * or any later version.
 *
 * In accordance with Section 7(e) of the license, the licensing of the Program
 * under the license does not imply a trademark license. Therefore any rights,
 * title and interest in our trademarks remain entirely with us.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the license for more details.
 *
 * You can be released from the requirements of the license by purchasing a
 * commercial license. Buying such a license is mandatory as soon as you develop
 * commercial activities involving this program without disclosing the source
 * code of your own applications
 */
#include "eventql/sql/svalue.h"
#include "eventql/server/sql/partition_cursor.h"
#include "eventql/transport/native/frames/error.h"
#include "eventql/transport/native/frames/query_remote.h"
#include "eventql/transport/native/frames/query_remote_result.h"
#include "eventql/db/database.h"

namespace eventql {

PartitionCursor::PartitionCursor(
    csql::Transaction* txn,
    csql::ExecutionContext* execution_context,
    RefPtr<Table> table,
    RefPtr<PartitionSnapshot> snap,
    RefPtr<csql::SequentialScanNode> stmt) :
    txn_(txn),
    execution_context_(execution_context),
    table_(table),
    snap_(snap),
    stmt_(stmt),
    cur_table_(0),
    fast_scan_enabled_(true) {
  if (stmt_->aggregationStrategy() != csql::AggregationStrategy::NO_AGGREGATION) {
    fast_scan_enabled_ = false;
  }
}

ReturnCode PartitionCursor::execute() {
  return ReturnCode::success();
}

ReturnCode PartitionCursor::nextBatch(
    csql::SVector* columns,
    size_t* nrows) {
  for (;;) {
    if (cur_scan_.get() == nullptr) {
      if (!openNextTable()) {
        break;
      }
    }

    auto rc = cur_scan_->nextBatch(columns, nrows);
    if (!rc.isSuccess()) {
      return rc;
    }

    if (!*nrows) {
      cur_scan_.reset(nullptr);
      continue;
    }

    return ReturnCode::success();
  }

  *nrows = 0;
  return ReturnCode::success();
}

bool PartitionCursor::openNextTable() {
  RefPtr<cstable::CSTableReader> cstable;
  String cstable_filename;
  RefPtr<cstable::ColumnReader> skip_col;

  cur_skiplist_.reset();

  bool needs_filter = true;
  switch (cur_table_) {
    case 0: {
      if (snap_->head_arena.get() &&
          snap_->head_arena->getCSTableFile()) {
        cur_skiplist_.reset(
            new PartitionArena::SkiplistReader(
                snap_->head_arena->getSkiplistReader()));
        cstable = cstable::CSTableReader::openFile(
            snap_->head_arena->getCSTableFile(),
            cur_skiplist_->size());
        cstable_filename = StringUtil::format(
            "memory://$0/$1/$2/head_arena",
            snap_->state.tsdb_namespace(),
            snap_->state.table_key(),
            snap_->key.toString());
        break;
      } else {
        ++cur_table_;
        /* fallthrough */
      }
    }

    case 1: {
      if (snap_->compacting_arena.get() &&
          snap_->compacting_arena->getCSTableFile()) {
        cur_skiplist_.reset(
            new PartitionArena::SkiplistReader(
                snap_->compacting_arena->getSkiplistReader()));
        cstable = cstable::CSTableReader::openFile(
            snap_->compacting_arena->getCSTableFile(),
            cur_skiplist_->size());
        cstable_filename = StringUtil::format(
            "memory://$0/$1/$2/compacting_arena",
            snap_->state.tsdb_namespace(),
            snap_->state.table_key(),
            snap_->key.toString());
        break;
      } else {
        ++cur_table_;
        /* fallthrough */
      }
    }

    default: {
      if (cur_table_ >= snap_->state.lsm_tables().size() + 2) {
        return false;
      }
      auto tblidx = snap_->state.lsm_tables().size() - (cur_table_ - 1);
      const auto& tbl = (snap_->state.lsm_tables().data())[tblidx];
      auto cstable_file = FileUtil::joinPaths(
          snap_->base_path,
          tbl->filename() + ".cst");
      cstable = cstable::CSTableReader::openFile(cstable_file);
      cstable_filename = cstable_file;
      if (tbl->has_skiplist()) {
        skip_col = cstable->getColumnReader("__lsm_skip");
      }

      if (!tbl->has_skiplist() && tblidx == 0 && id_set_.empty()) {
        needs_filter = false;
      }
      break;
    }
  }

  /* read filter */
  std::vector<bool> filter(cstable->numRecords(), true);
  if (needs_filter) {
    auto id_col = cstable->getColumnReader("__lsm_id");
    auto is_update_col = cstable->getColumnReader("__lsm_is_update");
    for (size_t i = 0; i < filter.size(); ++i) {
      uint64_t rlvl;
      uint64_t dlvl;

      String id_str;
      id_col->readString(&rlvl, &dlvl, &id_str);
      SHA1Hash id(id_str.data(), id_str.size());

      bool is_update;
      is_update_col->readBoolean(&rlvl, &dlvl, &is_update);

      bool skip = false;
      if (skip_col.get()) {
        skip_col->readBoolean(&rlvl, &dlvl, &skip);
      }

      if (cur_skiplist_) {
        skip = cur_skiplist_->readNext();
      }

      if (skip || id_set_.count(id) > 0) {
        filter[i] = false;
      } else {
        if (is_update) {
          id_set_.emplace(id);
        }

        filter[i] = true;
      }
    }
  }

  if (fast_scan_enabled_) {
    cur_scan_.reset(
        new csql::FastCSTableScan(
            txn_,
            execution_context_,
            stmt_,
            cstable,
            cstable_filename));
  } else {
    cur_scan_.reset(
        new csql::CSTableScan(
            txn_,
            execution_context_,
            stmt_,
            cstable,
            cstable_filename));
  }

  if (needs_filter) {
    cur_scan_->setFilter(std::move(filter));
  }

  ++cur_table_;

  auto rc = cur_scan_->execute();
  if (!rc.isSuccess()) {
    RAISE(kRuntimeError, rc.getMessage());
  }

  return true;
}

size_t PartitionCursor::getColumnCount() const {
  return stmt_->getNumComputedColumns();
}

csql::SType PartitionCursor::getColumnType(size_t idx) const {
  return stmt_->getColumnType(idx);
}

RemotePartitionCursor::RemotePartitionCursor(
    Session* session,
    csql::Transaction* txn,
    csql::ExecutionContext* execution_context,
    const std::string& database,
    RefPtr<csql::SequentialScanNode> stmt,
    const std::vector<std::string>& servers) :
    txn_(txn),
    execution_context_(execution_context),
    database_(database),
    stmt_(stmt),
    servers_(servers),
    ncols_(stmt->getNumComputedColumns()),
    row_buf_pos_(0),
    running_(false),
    done_(false),
    timeout_(
        session->getDatabaseContext()->config->getInt(
            "server.s2s_io_timeout").get()),
    client_(
        session->getDatabaseContext()->connection_pool,
        session->getDatabaseContext()->dns_cache,
        session->getDatabaseContext()->config->getInt(
            "server.s2s_io_timeout",
            0),
        session->getDatabaseContext()->config->getInt(
            "server.s2s_idle_timeout",
            0)) {}

ReturnCode RemotePartitionCursor::execute() {
  return ReturnCode::success();
}

ReturnCode RemotePartitionCursor::nextBatch(
    csql::SVector* output,
    size_t* output_len) {
  if (done_) {
    *output_len = 0;
    return ReturnCode::success();
  }

  if (running_) {
    auto rc = client_.sendFrame(EVQL_OP_QUERY_CONTINUE, 0, nullptr, 0);
    if (!rc.isSuccess()) {
      return rc;
    }
  } else {
    auto session = static_cast<eventql::Session*>(txn_->getUserData());
    auto cdir = session->getDatabaseContext()->config_directory;

    std::string qtree_coded;
    auto qtree_coded_os = StringOutputStream::fromString(&qtree_coded);
    csql::QueryTreeCoder qtree_coder(txn_);
    qtree_coder.encode(stmt_.get(), qtree_coded_os.get());

    native_transport::QueryRemoteFrame q_frame;
    q_frame.setEncodedQtree(std::move(qtree_coded));
    q_frame.setDatabase(session->getEffectiveNamespace());

    for (const auto& s : servers_) {
      auto server_cfg = cdir->getServerConfig(s);
      if (server_cfg.server_status() != SERVER_UP) {
        logError("evqld", "server is down: $0", s);
        continue;
      }

      auto rc = client_.connect(server_cfg.server_addr(), true);
      if (!rc.isSuccess()) {
        logError("evqld", "Remote SQL Error: $0", rc.getMessage());
        continue;
      }

      rc = client_.sendFrame(&q_frame, 0);
      if (rc.isSuccess()) {
        running_ = true;
      } else {
        logError("evqld", "Remote SQL Error: $0", rc.getMessage());
        continue;
      }
    }

    if (!running_) {
      return ReturnCode::error("ERUNTIME", "no server available for query");
    }
  }

  uint16_t ret_opcode = 0;
  uint16_t ret_flags;
  std::string ret_payload;
  while (ret_opcode != EVQL_OP_QUERY_REMOTE_RESULT) {
    auto rc = txn_->triggerHeartbeat();
    if (!rc.isSuccess()) {
      return rc;
    }

    rc = client_.recvFrame(&ret_opcode, &ret_flags, &ret_payload, timeout_);
    if (!rc.isSuccess()) {
      return rc;
    }

    if (ret_flags & EVQL_ENDOFREQUEST) {
      done_ = true;
    }

    switch (ret_opcode) {
      case EVQL_OP_HEARTBEAT:
        continue;
      case EVQL_OP_QUERY_REMOTE_RESULT:
        break;
      case EVQL_OP_ERROR: {
        native_transport::ErrorFrame eframe;
        eframe.parseFrom(ret_payload.data(), ret_payload.size());
        return ReturnCode::error("ERUNTIME", eframe.getError());
      }
      default:
        return ReturnCode::error("ERUNTIME", "invalid opcode");
    }
  }

  native_transport::QueryRemoteResultFrame r_frame;
  auto rc = r_frame.parseFrom(ret_payload.data(), ret_payload.size());

  if (r_frame.getColumnCount() != ncols_) {
    return ReturnCode::error("ERUNTIME", "invalid column count");
  }

  for (size_t i = 0; i < ncols_; ++i) {
    const auto& coldata = r_frame.getColumnData(i);
    output[i].append(coldata.data(), coldata.size());
  }

  *output_len = r_frame.getRowCount();
  return ReturnCode::success();
}

size_t RemotePartitionCursor::getColumnCount() const {
  return stmt_->getNumComputedColumns();
}

csql::SType RemotePartitionCursor::getColumnType(size_t idx) const {
  return stmt_->getColumnType(idx);
}

}
<|endoftext|>