{"action":"created","comment":{"url":"https://api.github.com/repos/druid-io/druid/pulls/comments/56013726","id":56013726,"diff_hunk":"@@ -0,0 +1,135 @@\n+/*\n+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one\n+ * or more contributor license agreements. See the NOTICE file\n+ * distributed with this work for additional information\n+ * regarding copyright ownership. Metamarkets licenses this file\n+ * to you under the Apache License, Version 2.0 (the\n+ * \"License\"); you may not use this file except in compliance\n+ * with the License. You may obtain a copy of the License at\n+ *\n+ * http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing,\n+ * software distributed under the License is distributed on an\n+ * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n+ * KIND, either express or implied. See the License for the\n+ * specific language governing permissions and limitations\n+ * under the License.\n+ */\n+\n+package io.druid.indexing.kafka;\n+\n+import com.fasterxml.jackson.annotation.JsonCreator;\n+import com.fasterxml.jackson.annotation.JsonProperty;\n+import io.druid.segment.IndexSpec;\n+import io.druid.segment.indexing.RealtimeTuningConfig;\n+import io.druid.segment.indexing.TuningConfig;\n+import io.druid.segment.realtime.appenderator.AppenderatorConfig;\n+import org.joda.time.Period;\n+\n+import java.io.File;\n+\n+public class KafkaTuningConfig implements TuningConfig, AppenderatorConfig\n+{\n+  private static final int DEFAULT_MAX_ROWS_PER_SEGMENT = 5_000_000;\n+\n+  private final int maxRowsInMemory;\n+  private final int maxRowsPerSegment;\n+  private final Period intermediatePersistPeriod;\n+  private final File basePersistDirectory;\n+  private final int maxPendingPersists;\n+  private final IndexSpec indexSpec;\n+  private final boolean buildV9Directly;\n+  private final boolean reportParseExceptions;\n+\n+  @JsonCreator\n+  public KafkaTuningConfig(\n+      @JsonProperty(\"maxRowsInMemory\") Integer maxRowsInMemory,\n+      @JsonProperty(\"maxRowsPerSegment\") Integer maxRowsPerSegment,\n+      @JsonProperty(\"intermediatePersistPeriod\") Period intermediatePersistPeriod,\n+      @JsonProperty(\"basePersistDirectory\") File basePersistDirectory,\n+      @JsonProperty(\"maxPendingPersists\") Integer maxPendingPersists,\n+      @JsonProperty(\"indexSpec\") IndexSpec indexSpec,\n+      @JsonProperty(\"buildV9Directly\") Boolean buildV9Directly,\n+      @JsonProperty(\"reportParseExceptions\") Boolean reportParseExceptions\n+  )\n+  {","path":"extensions/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaTuningConfig.java","position":null,"original_position":56,"commit_id":"f22fb2c2cf896b4b436e718548b825d7496aa470","original_commit_id":"111cfa698480e8923807af1c8b0039d247b85f5a","user":{"login":"himanshug","id":246308,"avatar_url":"https://avatars.githubusercontent.com/u/246308?v=3","gravatar_id":"","url":"https://api.github.com/users/himanshug","html_url":"https://github.com/himanshug","followers_url":"https://api.github.com/users/himanshug/followers","following_url":"https://api.github.com/users/himanshug/following{/other_user}","gists_url":"https://api.github.com/users/himanshug/gists{/gist_id}","starred_url":"https://api.github.com/users/himanshug/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/himanshug/subscriptions","organizations_url":"https://api.github.com/users/himanshug/orgs","repos_url":"https://api.github.com/users/himanshug/repos","events_url":"https://api.github.com/users/himanshug/events{/privacy}","received_events_url":"https://api.github.com/users/himanshug/received_events","type":"User","site_admin":false},"body":"we can keep it this way for now.","created_at":"2016-03-14T14:59:27Z","updated_at":"2016-03-14T14:59:27Z","html_url":"https://github.com/druid-io/druid/pull/2220#discussion_r56013726","pull_request_url":"https://api.github.com/repos/druid-io/druid/pulls/2220","_links":{"self":{"href":"https://api.github.com/repos/druid-io/druid/pulls/comments/56013726"},"html":{"href":"https://github.com/druid-io/druid/pull/2220#discussion_r56013726"},"pull_request":{"href":"https://api.github.com/repos/druid-io/druid/pulls/2220"}}},"pull_request":{"url":"https://api.github.com/repos/druid-io/druid/pulls/2220","id":55303849,"html_url":"https://github.com/druid-io/druid/pull/2220","diff_url":"https://github.com/druid-io/druid/pull/2220.diff","patch_url":"https://github.com/druid-io/druid/pull/2220.patch","issue_url":"https://api.github.com/repos/druid-io/druid/issues/2220","number":2220,"state":"open","locked":false,"title":"Appenderators, DataSource metadata, KafkaIndexTask","user":{"login":"gianm","id":1214075,"avatar_url":"https://avatars.githubusercontent.com/u/1214075?v=3","gravatar_id":"","url":"https://api.github.com/users/gianm","html_url":"https://github.com/gianm","followers_url":"https://api.github.com/users/gianm/followers","following_url":"https://api.github.com/users/gianm/following{/other_user}","gists_url":"https://api.github.com/users/gianm/gists{/gist_id}","starred_url":"https://api.github.com/users/gianm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gianm/subscriptions","organizations_url":"https://api.github.com/users/gianm/orgs","repos_url":"https://api.github.com/users/gianm/repos","events_url":"https://api.github.com/users/gianm/events{/privacy}","received_events_url":"https://api.github.com/users/gianm/received_events","type":"User","site_admin":false},"body":"See also #1642. FYI we are running this in our cluster as of 2016-03-02, things seem to work so far…\r\n\r\nSet of three related things (each one a separate commit):\r\n\r\n**kafka-indexing-service core extension**\r\nIncludes KafkaIndexTask, which reads a specific offset range from specific partitions, and can use dataSource metadata transactions to guarantee exactly-once ingestion.\r\n\r\nEach task has a finite lifecycle, so it is expected that a process will be supervising existing tasks and creating new ones when needed. @dclim is working on this process in https://github.com/dclim/druid/tree/kafka-supervisor.\r\n\r\nThis extension requires the other two features (DataSource metadata and Appenderators).\r\n\r\n**DataSource metadata**\r\nGeared towards supporting transactional inserts of new segments. This involves an interface `DataSourceMetadata` that allows combining of partially specified metadata (useful for partitioned ingestion). It also involves changes to the `SegmentInsertAction` to allow it to take a `startMetadata` and `endMetadata` for compare-and-swap.\r\n\r\nDataSource metadata is stored in a new \"dataSource\" table.\r\n\r\n**Appenderators**\r\nLike Plumbers, but different. Appenderators are a way of getting more control over the ingestion process than a Plumber allows. They are less ambitious than Plumbers, but more flexible. In particular, they offer facilities to deal with:\r\n\r\n- Indexing and persisting data\r\n- Checkpointing and restoring ingestion state\r\n- Merging and pushing segments to deep storage\r\n- Serving queries\r\n\r\nBut they do _not_ do any of these things:\r\n\r\n- Decide which rows to index and which to drop\r\n- Decide which segments to put your rows into\r\n- Decide when to push and publish segments\r\n- Monitor handoff\r\n\r\nSo you can think of Appenderators as a way of separating out the mechanical functionality of Plumbers from their decision-making processes. The idea is that existing Plumbers could be implemented using Appenderators, but you could also use them to implement workflows that the existing Plumbers can't support.\r\n\r\n**Discussion?**\r\nSome open questions (and reasons this is still marked Discuss),\r\n\r\n- Should RealtimePlumber be adapted to use an Appenderator? Pros of doing this are that the `AppenderatorImpl` actually has a lot of functionality and code overlap with RealtimePlumber (the query-runner stuff is particularly annoying since it is mostly similar but also kinda different). I think the main stumbling block is that the RealtimePlumber and the AppenderatorImpl have different persist directory layouts (mostly because the RealtimePlumber never has to have more than one shard per interval, but the AppenderatorImpl might) and so the code will need to be able to migrate existing data.\r\n\r\n- Should the other existing indexy things be adapted to use an Appenderator too? Like IndexTask and IndexGeneratorReducer.\r\n\r\nBTW, I think if we do either of these things, they should be in future PRs rather than this one.","created_at":"2016-01-07T06:18:17Z","updated_at":"2016-03-14T14:59:27Z","closed_at":null,"merged_at":null,"merge_commit_sha":"1db2d945cdce54d59220dc3143ccfa5b587e5394","assignee":null,"milestone":{"url":"https://api.github.com/repos/druid-io/druid/milestones/18","html_url":"https://github.com/druid-io/druid/milestones/0.9.1","labels_url":"https://api.github.com/repos/druid-io/druid/milestones/18/labels","id":1526819,"number":18,"title":"0.9.1","description":"","creator":{"login":"fjy","id":428325,"avatar_url":"https://avatars.githubusercontent.com/u/428325?v=3","gravatar_id":"","url":"https://api.github.com/users/fjy","html_url":"https://github.com/fjy","followers_url":"https://api.github.com/users/fjy/followers","following_url":"https://api.github.com/users/fjy/following{/other_user}","gists_url":"https://api.github.com/users/fjy/gists{/gist_id}","starred_url":"https://api.github.com/users/fjy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fjy/subscriptions","organizations_url":"https://api.github.com/users/fjy/orgs","repos_url":"https://api.github.com/users/fjy/repos","events_url":"https://api.github.com/users/fjy/events{/privacy}","received_events_url":"https://api.github.com/users/fjy/received_events","type":"User","site_admin":false},"open_issues":33,"closed_issues":59,"state":"open","created_at":"2016-01-20T19:04:22Z","updated_at":"2016-03-13T18:16:24Z","due_on":null,"closed_at":null},"commits_url":"https://api.github.com/repos/druid-io/druid/pulls/2220/commits","review_comments_url":"https://api.github.com/repos/druid-io/druid/pulls/2220/comments","review_comment_url":"https://api.github.com/repos/druid-io/druid/pulls/comments{/number}","comments_url":"https://api.github.com/repos/druid-io/druid/issues/2220/comments","statuses_url":"https://api.github.com/repos/druid-io/druid/statuses/f22fb2c2cf896b4b436e718548b825d7496aa470","head":{"label":"gianm:appenderator-kafka","ref":"appenderator-kafka","sha":"f22fb2c2cf896b4b436e718548b825d7496aa470","user":{"login":"gianm","id":1214075,"avatar_url":"https://avatars.githubusercontent.com/u/1214075?v=3","gravatar_id":"","url":"https://api.github.com/users/gianm","html_url":"https://github.com/gianm","followers_url":"https://api.github.com/users/gianm/followers","following_url":"https://api.github.com/users/gianm/following{/other_user}","gists_url":"https://api.github.com/users/gianm/gists{/gist_id}","starred_url":"https://api.github.com/users/gianm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gianm/subscriptions","organizations_url":"https://api.github.com/users/gianm/orgs","repos_url":"https://api.github.com/users/gianm/repos","events_url":"https://api.github.com/users/gianm/events{/privacy}","received_events_url":"https://api.github.com/users/gianm/received_events","type":"User","site_admin":false},"repo":{"id":28350072,"name":"druid","full_name":"gianm/druid","owner":{"login":"gianm","id":1214075,"avatar_url":"https://avatars.githubusercontent.com/u/1214075?v=3","gravatar_id":"","url":"https://api.github.com/users/gianm","html_url":"https://github.com/gianm","followers_url":"https://api.github.com/users/gianm/followers","following_url":"https://api.github.com/users/gianm/following{/other_user}","gists_url":"https://api.github.com/users/gianm/gists{/gist_id}","starred_url":"https://api.github.com/users/gianm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gianm/subscriptions","organizations_url":"https://api.github.com/users/gianm/orgs","repos_url":"https://api.github.com/users/gianm/repos","events_url":"https://api.github.com/users/gianm/events{/privacy}","received_events_url":"https://api.github.com/users/gianm/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/gianm/druid","description":"Real²time Exploratory Analytics on Large Datasets","fork":true,"url":"https://api.github.com/repos/gianm/druid","forks_url":"https://api.github.com/repos/gianm/druid/forks","keys_url":"https://api.github.com/repos/gianm/druid/keys{/key_id}","collaborators_url":"https://api.github.com/repos/gianm/druid/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/gianm/druid/teams","hooks_url":"https://api.github.com/repos/gianm/druid/hooks","issue_events_url":"https://api.github.com/repos/gianm/druid/issues/events{/number}","events_url":"https://api.github.com/repos/gianm/druid/events","assignees_url":"https://api.github.com/repos/gianm/druid/assignees{/user}","branches_url":"https://api.github.com/repos/gianm/druid/branches{/branch}","tags_url":"https://api.github.com/repos/gianm/druid/tags","blobs_url":"https://api.github.com/repos/gianm/druid/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/gianm/druid/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/gianm/druid/git/refs{/sha}","trees_url":"https://api.github.com/repos/gianm/druid/git/trees{/sha}","statuses_url":"https://api.github.com/repos/gianm/druid/statuses/{sha}","languages_url":"https://api.github.com/repos/gianm/druid/languages","stargazers_url":"https://api.github.com/repos/gianm/druid/stargazers","contributors_url":"https://api.github.com/repos/gianm/druid/contributors","subscribers_url":"https://api.github.com/repos/gianm/druid/subscribers","subscription_url":"https://api.github.com/repos/gianm/druid/subscription","commits_url":"https://api.github.com/repos/gianm/druid/commits{/sha}","git_commits_url":"https://api.github.com/repos/gianm/druid/git/commits{/sha}","comments_url":"https://api.github.com/repos/gianm/druid/comments{/number}","issue_comment_url":"https://api.github.com/repos/gianm/druid/issues/comments{/number}","contents_url":"https://api.github.com/repos/gianm/druid/contents/{+path}","compare_url":"https://api.github.com/repos/gianm/druid/compare/{base}...{head}","merges_url":"https://api.github.com/repos/gianm/druid/merges","archive_url":"https://api.github.com/repos/gianm/druid/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/gianm/druid/downloads","issues_url":"https://api.github.com/repos/gianm/druid/issues{/number}","pulls_url":"https://api.github.com/repos/gianm/druid/pulls{/number}","milestones_url":"https://api.github.com/repos/gianm/druid/milestones{/number}","notifications_url":"https://api.github.com/repos/gianm/druid/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/gianm/druid/labels{/name}","releases_url":"https://api.github.com/repos/gianm/druid/releases{/id}","deployments_url":"https://api.github.com/repos/gianm/druid/deployments","created_at":"2014-12-22T17:32:01Z","updated_at":"2016-01-18T01:27:33Z","pushed_at":"2016-03-12T01:37:28Z","git_url":"git://github.com/gianm/druid.git","ssh_url":"git@github.com:gianm/druid.git","clone_url":"https://github.com/gianm/druid.git","svn_url":"https://github.com/gianm/druid","homepage":"http://druid.io","size":104076,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"druid-io:master","ref":"master","sha":"1e49092ce767239572fd00dc0fd4e38bf648d977","user":{"login":"druid-io","id":4203474,"avatar_url":"https://avatars.githubusercontent.com/u/4203474?v=3","gravatar_id":"","url":"https://api.github.com/users/druid-io","html_url":"https://github.com/druid-io","followers_url":"https://api.github.com/users/druid-io/followers","following_url":"https://api.github.com/users/druid-io/following{/other_user}","gists_url":"https://api.github.com/users/druid-io/gists{/gist_id}","starred_url":"https://api.github.com/users/druid-io/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/druid-io/subscriptions","organizations_url":"https://api.github.com/users/druid-io/orgs","repos_url":"https://api.github.com/users/druid-io/repos","events_url":"https://api.github.com/users/druid-io/events{/privacy}","received_events_url":"https://api.github.com/users/druid-io/received_events","type":"Organization","site_admin":false},"repo":{"id":6358188,"name":"druid","full_name":"druid-io/druid","owner":{"login":"druid-io","id":4203474,"avatar_url":"https://avatars.githubusercontent.com/u/4203474?v=3","gravatar_id":"","url":"https://api.github.com/users/druid-io","html_url":"https://github.com/druid-io","followers_url":"https://api.github.com/users/druid-io/followers","following_url":"https://api.github.com/users/druid-io/following{/other_user}","gists_url":"https://api.github.com/users/druid-io/gists{/gist_id}","starred_url":"https://api.github.com/users/druid-io/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/druid-io/subscriptions","organizations_url":"https://api.github.com/users/druid-io/orgs","repos_url":"https://api.github.com/users/druid-io/repos","events_url":"https://api.github.com/users/druid-io/events{/privacy}","received_events_url":"https://api.github.com/users/druid-io/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/druid-io/druid","description":"Column oriented distributed data store ideal for powering interactive applications","fork":false,"url":"https://api.github.com/repos/druid-io/druid","forks_url":"https://api.github.com/repos/druid-io/druid/forks","keys_url":"https://api.github.com/repos/druid-io/druid/keys{/key_id}","collaborators_url":"https://api.github.com/repos/druid-io/druid/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/druid-io/druid/teams","hooks_url":"https://api.github.com/repos/druid-io/druid/hooks","issue_events_url":"https://api.github.com/repos/druid-io/druid/issues/events{/number}","events_url":"https://api.github.com/repos/druid-io/druid/events","assignees_url":"https://api.github.com/repos/druid-io/druid/assignees{/user}","branches_url":"https://api.github.com/repos/druid-io/druid/branches{/branch}","tags_url":"https://api.github.com/repos/druid-io/druid/tags","blobs_url":"https://api.github.com/repos/druid-io/druid/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/druid-io/druid/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/druid-io/druid/git/refs{/sha}","trees_url":"https://api.github.com/repos/druid-io/druid/git/trees{/sha}","statuses_url":"https://api.github.com/repos/druid-io/druid/statuses/{sha}","languages_url":"https://api.github.com/repos/druid-io/druid/languages","stargazers_url":"https://api.github.com/repos/druid-io/druid/stargazers","contributors_url":"https://api.github.com/repos/druid-io/druid/contributors","subscribers_url":"https://api.github.com/repos/druid-io/druid/subscribers","subscription_url":"https://api.github.com/repos/druid-io/druid/subscription","commits_url":"https://api.github.com/repos/druid-io/druid/commits{/sha}","git_commits_url":"https://api.github.com/repos/druid-io/druid/git/commits{/sha}","comments_url":"https://api.github.com/repos/druid-io/druid/comments{/number}","issue_comment_url":"https://api.github.com/repos/druid-io/druid/issues/comments{/number}","contents_url":"https://api.github.com/repos/druid-io/druid/contents/{+path}","compare_url":"https://api.github.com/repos/druid-io/druid/compare/{base}...{head}","merges_url":"https://api.github.com/repos/druid-io/druid/merges","archive_url":"https://api.github.com/repos/druid-io/druid/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/druid-io/druid/downloads","issues_url":"https://api.github.com/repos/druid-io/druid/issues{/number}","pulls_url":"https://api.github.com/repos/druid-io/druid/pulls{/number}","milestones_url":"https://api.github.com/repos/druid-io/druid/milestones{/number}","notifications_url":"https://api.github.com/repos/druid-io/druid/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/druid-io/druid/labels{/name}","releases_url":"https://api.github.com/repos/druid-io/druid/releases{/id}","deployments_url":"https://api.github.com/repos/druid-io/druid/deployments","created_at":"2012-10-23T19:08:07Z","updated_at":"2016-03-14T09:37:36Z","pushed_at":"2016-03-14T14:57:07Z","git_url":"git://github.com/druid-io/druid.git","ssh_url":"git@github.com:druid-io/druid.git","clone_url":"https://github.com/druid-io/druid.git","svn_url":"https://github.com/druid-io/druid","homepage":"http://druid.io","size":127307,"stargazers_count":2538,"watchers_count":2538,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":608,"mirror_url":null,"open_issues_count":364,"forks":608,"open_issues":364,"watchers":2538,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/druid-io/druid/pulls/2220"},"html":{"href":"https://github.com/druid-io/druid/pull/2220"},"issue":{"href":"https://api.github.com/repos/druid-io/druid/issues/2220"},"comments":{"href":"https://api.github.com/repos/druid-io/druid/issues/2220/comments"},"review_comments":{"href":"https://api.github.com/repos/druid-io/druid/pulls/2220/comments"},"review_comment":{"href":"https://api.github.com/repos/druid-io/druid/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/druid-io/druid/pulls/2220/commits"},"statuses":{"href":"https://api.github.com/repos/druid-io/druid/statuses/f22fb2c2cf896b4b436e718548b825d7496aa470"}}}}