{"action":"created","comment":{"url":"https://api.github.com/repos/rackerlabs/blueflood/pulls/comments/52177611","id":52177611,"diff_hunk":"@@ -0,0 +1,204 @@\n+package com.rackspacecloud.blueflood.io;\n+\n+import java.util.*;\n+\n+/**\n+ * When we ingest these below metrics, ES generates the following indexes\n+ *\n+ *     metric           indexes\n+ *   -----------        -------\n+ *   foo.bar.baz.aux -> [foo, foo.bar, foo.bar.baz, foo.bar.baz.aux, bar, baz, aux]\n+ *   foo.bar.baz     -> [foo, foo.bar, foo.bar.baz, bar, baz]\n+ *\n+ * If we request ES to return us upto next two levels of the metric indexes for foo.bar,\n+ * we get this below response from ES where key is the metric index and doc_count is\n+ * the number of documents(think metric names) it points to.\n+ *\n+ * \"buckets\" : [ {\n+ *     \"key\" : \"foo.bar\",\n+ *     \"doc_count\" : 2\n+ *   }, {\n+ *     \"key\" : \"foo.bar.baz\",\n+ *     \"doc_count\" : 2\n+ *   }, {\n+ *     \"key\" : \"foo.bar.baz.aux\",\n+ *     \"doc_count\" : 1\n+ * } ]\n+ *\n+ * If we feed each metric index to this class by calling the add(metricIndex, docCount) method,\n+ * this class analyzes the data and provides methods to retrieve different aspects of this data\n+ * with respect to baseLevel(if set to 2 with above data, uses foo.bar as reference).\n+ *\n+ */\n+public class MetricIndexData {\n+\n+    private static class MetricIndexDocCount {\n+\n+        private long actualDocCount;\n+        private long childrenTotalDocCount;\n+\n+        public MetricIndexDocCount(long actualDocCount, long childrenTotalDocCount) {\n+            this.actualDocCount = actualDocCount;\n+            this.childrenTotalDocCount = childrenTotalDocCount;\n+        }\n+\n+        public void setActualDocCount(long actualDocCount) {\n+            this.actualDocCount = actualDocCount;\n+        }\n+\n+        public void addChildrenTotalDocCount(long childrenDocCount) {\n+            this.childrenTotalDocCount += childrenDocCount;\n+        }\n+    }\n+\n+    private final int baseLevel;\n+\n+    //contains next level of tokens after baseLevel, which also have subsequent next level\n+    private final Set<String> tokensWithNextLevelSet = new LinkedHashSet<String>();\n+\n+    //contains all metric indexes which are of the same length as baseLevel\n+    final Map<String, MetricIndexDocCount> metricIndexSameLevelMap = new HashMap<String, MetricIndexDocCount>();\n+\n+    //contains all metric indexes which are more than the length of baseLevel by one.\n+    final Map<String, MetricIndexDocCount> metricIndexNextLevelMap = new HashMap<String, MetricIndexDocCount>();\n+\n+    public MetricIndexData(int baseLevel) {\n+        this.baseLevel = baseLevel;\n+    }\n+\n+    /**\n+     * For a given metricIndex and docCount, classifies the data with respect to baseLevel\n+     * and stores it accordingly.\n+     *\n+     * @param metricIndex\n+     * @param docCount is document count\n+     */\n+    public void add(String metricIndex, long docCount) {\n+\n+        final String[] tokens = metricIndex.split(AbstractElasticIO.REGEX_TOKEN_DELIMTER);\n+\n+        /**\n+         *\n+         * For the ES response shown in class description, for a baseLevel of 2,\n+         * the data is classified as follows\n+         *\n+         * tokensWithNextLevelSet    -> {bar}               (tokens after baseLevel which also have a subsequent level)\n+         *\n+         * Both these maps maintain data in the form {metricIndex, (actualDocCount, childrenTotalDocCount)}\n+         *\n+         * metricIndexSameLevelMap   -> {foo.bar -> (2, 2)}     (all indexes which are of same length as baseLevel)","path":"blueflood-elasticsearch/src/main/java/com/rackspacecloud/blueflood/io/MetricIndexData.java","position":89,"original_position":89,"commit_id":"be26a9f8947ad67bdb722d5eb9483f2f44db39d7","original_commit_id":"18a6a1e4a4268220bc2dbf925784965f8c953697","user":{"login":"usnavi","id":1426961,"avatar_url":"https://avatars.githubusercontent.com/u/1426961?v=3","gravatar_id":"","url":"https://api.github.com/users/usnavi","html_url":"https://github.com/usnavi","followers_url":"https://api.github.com/users/usnavi/followers","following_url":"https://api.github.com/users/usnavi/following{/other_user}","gists_url":"https://api.github.com/users/usnavi/gists{/gist_id}","starred_url":"https://api.github.com/users/usnavi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/usnavi/subscriptions","organizations_url":"https://api.github.com/users/usnavi/orgs","repos_url":"https://api.github.com/users/usnavi/repos","events_url":"https://api.github.com/users/usnavi/events{/privacy}","received_events_url":"https://api.github.com/users/usnavi/received_events","type":"User","site_admin":false},"body":"OK, i guess with the docCount being a sum of all paths, including grandchildren, that's where I've gotten confused.","created_at":"2016-02-08T14:53:39Z","updated_at":"2016-02-08T14:53:39Z","html_url":"https://github.com/rackerlabs/blueflood/pull/610#discussion_r52177611","pull_request_url":"https://api.github.com/repos/rackerlabs/blueflood/pulls/610","_links":{"self":{"href":"https://api.github.com/repos/rackerlabs/blueflood/pulls/comments/52177611"},"html":{"href":"https://github.com/rackerlabs/blueflood/pull/610#discussion_r52177611"},"pull_request":{"href":"https://api.github.com/repos/rackerlabs/blueflood/pulls/610"}}},"pull_request":{"url":"https://api.github.com/repos/rackerlabs/blueflood/pulls/610","id":58056875,"html_url":"https://github.com/rackerlabs/blueflood/pull/610","diff_url":"https://github.com/rackerlabs/blueflood/pull/610.diff","patch_url":"https://github.com/rackerlabs/blueflood/pull/610.patch","issue_url":"https://api.github.com/repos/rackerlabs/blueflood/issues/610","number":610,"state":"open","locked":false,"title":"Finder performance issues","user":{"login":"ChandraAddala","id":1525859,"avatar_url":"https://avatars.githubusercontent.com/u/1525859?v=3","gravatar_id":"","url":"https://api.github.com/users/ChandraAddala","html_url":"https://github.com/ChandraAddala","followers_url":"https://api.github.com/users/ChandraAddala/followers","following_url":"https://api.github.com/users/ChandraAddala/following{/other_user}","gists_url":"https://api.github.com/users/ChandraAddala/gists{/gist_id}","starred_url":"https://api.github.com/users/ChandraAddala/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ChandraAddala/subscriptions","organizations_url":"https://api.github.com/users/ChandraAddala/orgs","repos_url":"https://api.github.com/users/ChandraAddala/repos","events_url":"https://api.github.com/users/ChandraAddala/events{/privacy}","received_events_url":"https://api.github.com/users/ChandraAddala/received_events","type":"User","site_admin":false},"body":"    Finder performance issues\r\n     - Adding a new API /next_token to grab tokens of metric name\r\n     - Refactored common code of ElasticIO and EnumElasticIO to AbstractElasticIO\r\n     - Added tests","created_at":"2016-02-02T21:58:03Z","updated_at":"2016-02-08T14:53:39Z","closed_at":null,"merged_at":null,"merge_commit_sha":"fb66a54e6b380991930a989786184141fd1986d1","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/rackerlabs/blueflood/pulls/610/commits","review_comments_url":"https://api.github.com/repos/rackerlabs/blueflood/pulls/610/comments","review_comment_url":"https://api.github.com/repos/rackerlabs/blueflood/pulls/comments{/number}","comments_url":"https://api.github.com/repos/rackerlabs/blueflood/issues/610/comments","statuses_url":"https://api.github.com/repos/rackerlabs/blueflood/statuses/be26a9f8947ad67bdb722d5eb9483f2f44db39d7","head":{"label":"ChandraAddala:new-api-next-token","ref":"new-api-next-token","sha":"be26a9f8947ad67bdb722d5eb9483f2f44db39d7","user":{"login":"ChandraAddala","id":1525859,"avatar_url":"https://avatars.githubusercontent.com/u/1525859?v=3","gravatar_id":"","url":"https://api.github.com/users/ChandraAddala","html_url":"https://github.com/ChandraAddala","followers_url":"https://api.github.com/users/ChandraAddala/followers","following_url":"https://api.github.com/users/ChandraAddala/following{/other_user}","gists_url":"https://api.github.com/users/ChandraAddala/gists{/gist_id}","starred_url":"https://api.github.com/users/ChandraAddala/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ChandraAddala/subscriptions","organizations_url":"https://api.github.com/users/ChandraAddala/orgs","repos_url":"https://api.github.com/users/ChandraAddala/repos","events_url":"https://api.github.com/users/ChandraAddala/events{/privacy}","received_events_url":"https://api.github.com/users/ChandraAddala/received_events","type":"User","site_admin":false},"repo":{"id":45928272,"name":"blueflood","full_name":"ChandraAddala/blueflood","owner":{"login":"ChandraAddala","id":1525859,"avatar_url":"https://avatars.githubusercontent.com/u/1525859?v=3","gravatar_id":"","url":"https://api.github.com/users/ChandraAddala","html_url":"https://github.com/ChandraAddala","followers_url":"https://api.github.com/users/ChandraAddala/followers","following_url":"https://api.github.com/users/ChandraAddala/following{/other_user}","gists_url":"https://api.github.com/users/ChandraAddala/gists{/gist_id}","starred_url":"https://api.github.com/users/ChandraAddala/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ChandraAddala/subscriptions","organizations_url":"https://api.github.com/users/ChandraAddala/orgs","repos_url":"https://api.github.com/users/ChandraAddala/repos","events_url":"https://api.github.com/users/ChandraAddala/events{/privacy}","received_events_url":"https://api.github.com/users/ChandraAddala/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/ChandraAddala/blueflood","description":"A distributed system designed to ingest and process time series data","fork":true,"url":"https://api.github.com/repos/ChandraAddala/blueflood","forks_url":"https://api.github.com/repos/ChandraAddala/blueflood/forks","keys_url":"https://api.github.com/repos/ChandraAddala/blueflood/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ChandraAddala/blueflood/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ChandraAddala/blueflood/teams","hooks_url":"https://api.github.com/repos/ChandraAddala/blueflood/hooks","issue_events_url":"https://api.github.com/repos/ChandraAddala/blueflood/issues/events{/number}","events_url":"https://api.github.com/repos/ChandraAddala/blueflood/events","assignees_url":"https://api.github.com/repos/ChandraAddala/blueflood/assignees{/user}","branches_url":"https://api.github.com/repos/ChandraAddala/blueflood/branches{/branch}","tags_url":"https://api.github.com/repos/ChandraAddala/blueflood/tags","blobs_url":"https://api.github.com/repos/ChandraAddala/blueflood/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ChandraAddala/blueflood/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ChandraAddala/blueflood/git/refs{/sha}","trees_url":"https://api.github.com/repos/ChandraAddala/blueflood/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ChandraAddala/blueflood/statuses/{sha}","languages_url":"https://api.github.com/repos/ChandraAddala/blueflood/languages","stargazers_url":"https://api.github.com/repos/ChandraAddala/blueflood/stargazers","contributors_url":"https://api.github.com/repos/ChandraAddala/blueflood/contributors","subscribers_url":"https://api.github.com/repos/ChandraAddala/blueflood/subscribers","subscription_url":"https://api.github.com/repos/ChandraAddala/blueflood/subscription","commits_url":"https://api.github.com/repos/ChandraAddala/blueflood/commits{/sha}","git_commits_url":"https://api.github.com/repos/ChandraAddala/blueflood/git/commits{/sha}","comments_url":"https://api.github.com/repos/ChandraAddala/blueflood/comments{/number}","issue_comment_url":"https://api.github.com/repos/ChandraAddala/blueflood/issues/comments{/number}","contents_url":"https://api.github.com/repos/ChandraAddala/blueflood/contents/{+path}","compare_url":"https://api.github.com/repos/ChandraAddala/blueflood/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ChandraAddala/blueflood/merges","archive_url":"https://api.github.com/repos/ChandraAddala/blueflood/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ChandraAddala/blueflood/downloads","issues_url":"https://api.github.com/repos/ChandraAddala/blueflood/issues{/number}","pulls_url":"https://api.github.com/repos/ChandraAddala/blueflood/pulls{/number}","milestones_url":"https://api.github.com/repos/ChandraAddala/blueflood/milestones{/number}","notifications_url":"https://api.github.com/repos/ChandraAddala/blueflood/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ChandraAddala/blueflood/labels{/name}","releases_url":"https://api.github.com/repos/ChandraAddala/blueflood/releases{/id}","deployments_url":"https://api.github.com/repos/ChandraAddala/blueflood/deployments","created_at":"2015-11-10T17:25:47Z","updated_at":"2016-02-03T15:45:28Z","pushed_at":"2016-02-07T21:55:54Z","git_url":"git://github.com/ChandraAddala/blueflood.git","ssh_url":"git@github.com:ChandraAddala/blueflood.git","clone_url":"https://github.com/ChandraAddala/blueflood.git","svn_url":"https://github.com/ChandraAddala/blueflood","homepage":"http://www.blueflood.io","size":8985,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"rackerlabs:master","ref":"master","sha":"ecb6c09bf4cec7aff9384c365a1fd3631ea3d362","user":{"login":"rackerlabs","id":84293,"avatar_url":"https://avatars.githubusercontent.com/u/84293?v=3","gravatar_id":"","url":"https://api.github.com/users/rackerlabs","html_url":"https://github.com/rackerlabs","followers_url":"https://api.github.com/users/rackerlabs/followers","following_url":"https://api.github.com/users/rackerlabs/following{/other_user}","gists_url":"https://api.github.com/users/rackerlabs/gists{/gist_id}","starred_url":"https://api.github.com/users/rackerlabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rackerlabs/subscriptions","organizations_url":"https://api.github.com/users/rackerlabs/orgs","repos_url":"https://api.github.com/users/rackerlabs/repos","events_url":"https://api.github.com/users/rackerlabs/events{/privacy}","received_events_url":"https://api.github.com/users/rackerlabs/received_events","type":"Organization","site_admin":false},"repo":{"id":10081109,"name":"blueflood","full_name":"rackerlabs/blueflood","owner":{"login":"rackerlabs","id":84293,"avatar_url":"https://avatars.githubusercontent.com/u/84293?v=3","gravatar_id":"","url":"https://api.github.com/users/rackerlabs","html_url":"https://github.com/rackerlabs","followers_url":"https://api.github.com/users/rackerlabs/followers","following_url":"https://api.github.com/users/rackerlabs/following{/other_user}","gists_url":"https://api.github.com/users/rackerlabs/gists{/gist_id}","starred_url":"https://api.github.com/users/rackerlabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rackerlabs/subscriptions","organizations_url":"https://api.github.com/users/rackerlabs/orgs","repos_url":"https://api.github.com/users/rackerlabs/repos","events_url":"https://api.github.com/users/rackerlabs/events{/privacy}","received_events_url":"https://api.github.com/users/rackerlabs/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/rackerlabs/blueflood","description":"A distributed system designed to ingest and process time series data","fork":false,"url":"https://api.github.com/repos/rackerlabs/blueflood","forks_url":"https://api.github.com/repos/rackerlabs/blueflood/forks","keys_url":"https://api.github.com/repos/rackerlabs/blueflood/keys{/key_id}","collaborators_url":"https://api.github.com/repos/rackerlabs/blueflood/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/rackerlabs/blueflood/teams","hooks_url":"https://api.github.com/repos/rackerlabs/blueflood/hooks","issue_events_url":"https://api.github.com/repos/rackerlabs/blueflood/issues/events{/number}","events_url":"https://api.github.com/repos/rackerlabs/blueflood/events","assignees_url":"https://api.github.com/repos/rackerlabs/blueflood/assignees{/user}","branches_url":"https://api.github.com/repos/rackerlabs/blueflood/branches{/branch}","tags_url":"https://api.github.com/repos/rackerlabs/blueflood/tags","blobs_url":"https://api.github.com/repos/rackerlabs/blueflood/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/rackerlabs/blueflood/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/rackerlabs/blueflood/git/refs{/sha}","trees_url":"https://api.github.com/repos/rackerlabs/blueflood/git/trees{/sha}","statuses_url":"https://api.github.com/repos/rackerlabs/blueflood/statuses/{sha}","languages_url":"https://api.github.com/repos/rackerlabs/blueflood/languages","stargazers_url":"https://api.github.com/repos/rackerlabs/blueflood/stargazers","contributors_url":"https://api.github.com/repos/rackerlabs/blueflood/contributors","subscribers_url":"https://api.github.com/repos/rackerlabs/blueflood/subscribers","subscription_url":"https://api.github.com/repos/rackerlabs/blueflood/subscription","commits_url":"https://api.github.com/repos/rackerlabs/blueflood/commits{/sha}","git_commits_url":"https://api.github.com/repos/rackerlabs/blueflood/git/commits{/sha}","comments_url":"https://api.github.com/repos/rackerlabs/blueflood/comments{/number}","issue_comment_url":"https://api.github.com/repos/rackerlabs/blueflood/issues/comments{/number}","contents_url":"https://api.github.com/repos/rackerlabs/blueflood/contents/{+path}","compare_url":"https://api.github.com/repos/rackerlabs/blueflood/compare/{base}...{head}","merges_url":"https://api.github.com/repos/rackerlabs/blueflood/merges","archive_url":"https://api.github.com/repos/rackerlabs/blueflood/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/rackerlabs/blueflood/downloads","issues_url":"https://api.github.com/repos/rackerlabs/blueflood/issues{/number}","pulls_url":"https://api.github.com/repos/rackerlabs/blueflood/pulls{/number}","milestones_url":"https://api.github.com/repos/rackerlabs/blueflood/milestones{/number}","notifications_url":"https://api.github.com/repos/rackerlabs/blueflood/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/rackerlabs/blueflood/labels{/name}","releases_url":"https://api.github.com/repos/rackerlabs/blueflood/releases{/id}","deployments_url":"https://api.github.com/repos/rackerlabs/blueflood/deployments","created_at":"2013-05-15T14:50:39Z","updated_at":"2016-01-29T16:46:37Z","pushed_at":"2016-02-07T21:55:54Z","git_url":"git://github.com/rackerlabs/blueflood.git","ssh_url":"git@github.com:rackerlabs/blueflood.git","clone_url":"https://github.com/rackerlabs/blueflood.git","svn_url":"https://github.com/rackerlabs/blueflood","homepage":"http://www.blueflood.io","size":12246,"stargazers_count":405,"watchers_count":405,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":54,"mirror_url":null,"open_issues_count":30,"forks":54,"open_issues":30,"watchers":405,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/rackerlabs/blueflood/pulls/610"},"html":{"href":"https://github.com/rackerlabs/blueflood/pull/610"},"issue":{"href":"https://api.github.com/repos/rackerlabs/blueflood/issues/610"},"comments":{"href":"https://api.github.com/repos/rackerlabs/blueflood/issues/610/comments"},"review_comments":{"href":"https://api.github.com/repos/rackerlabs/blueflood/pulls/610/comments"},"review_comment":{"href":"https://api.github.com/repos/rackerlabs/blueflood/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/rackerlabs/blueflood/pulls/610/commits"},"statuses":{"href":"https://api.github.com/repos/rackerlabs/blueflood/statuses/be26a9f8947ad67bdb722d5eb9483f2f44db39d7"}}}}