{"action":"created","comment":{"url":"https://api.github.com/repos/rewiko/curator/pulls/comments/167272577","pull_request_review_id":95458222,"id":167272577,"diff_hunk":"@@ -134,10 +134,39 @@ def iterate_over_stats(stats):\n         if working_list:\n             index_lists = chunk_index_list(working_list)\n             for l in index_lists:\n-                iterate_over_stats(\n-                    self.client.indices.stats(index=to_csv(l),\n-                    metric='store,docs')\n-                )\n+                stats_result = {}\n+\n+                try:\n+                    stats_result.update(self.get_indices_stats(l))\n+                except elasticsearch.ElasticsearchException as err:\n+                    if err.status_code == 413:\n+                        self.loggit.debug('Huge Payload 413 Error - Trying to get information with multiple requests')\n+                        stats_result = {}\n+                        stats_result.update(self.bulk_queries(l, self.get_indices_stats))\n+                        pass\n+\n+                iterate_over_stats(stats_result)\n+\n+    def get_indices_stats(self, data):\n+        return self.client.indices.stats(index=to_csv(data), metric='store,docs')\n+\n+    def bulk_queries(self, data, exec_func):\n+        slice_number = 10\n+        query_result = {}\n+        loop_number = round(len(data)/slice_number) if round(len(data)/slice_number) > 0 else 1\n+        self.loggit.debug(\"Bulk Queries - number requests created: {0}\".format(loop_number))\n+\n+        for num in range(0, loop_number):\n+            if num == (loop_number-1):\n+                data_sliced = data[num*slice_number:]\n+            else:\n+                data_sliced = data[num*slice_number:(num+1)*slice_number]\n+            query_result.update(exec_func(data_sliced))\n+\n+        return query_result\n+\n+    def get_cluster_state(self, data):\n+        return self.client.cluster.state(index=to_csv(data),metric='metadata')['metadata']['indices']","path":"curator/indexlist.py","position":40,"original_position":40,"commit_id":"25117d3cc1fc2803d31e1992c4857bf01cee263b","original_commit_id":"25117d3cc1fc2803d31e1992c4857bf01cee263b","user":{"login":"MalsR","id":1785272,"avatar_url":"https://avatars3.githubusercontent.com/u/1785272?v=4","gravatar_id":"","url":"https://api.github.com/users/MalsR","html_url":"https://github.com/MalsR","followers_url":"https://api.github.com/users/MalsR/followers","following_url":"https://api.github.com/users/MalsR/following{/other_user}","gists_url":"https://api.github.com/users/MalsR/gists{/gist_id}","starred_url":"https://api.github.com/users/MalsR/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/MalsR/subscriptions","organizations_url":"https://api.github.com/users/MalsR/orgs","repos_url":"https://api.github.com/users/MalsR/repos","events_url":"https://api.github.com/users/MalsR/events{/privacy}","received_events_url":"https://api.github.com/users/MalsR/received_events","type":"User","site_admin":false},"body":"I'm nitpicking space after comma `,metric='metadata'`","created_at":"2018-02-09T16:14:08Z","updated_at":"2018-02-09T16:25:12Z","html_url":"https://github.com/rewiko/curator/pull/1#discussion_r167272577","pull_request_url":"https://api.github.com/repos/rewiko/curator/pulls/1","author_association":"NONE","_links":{"self":{"href":"https://api.github.com/repos/rewiko/curator/pulls/comments/167272577"},"html":{"href":"https://github.com/rewiko/curator/pull/1#discussion_r167272577"},"pull_request":{"href":"https://api.github.com/repos/rewiko/curator/pulls/1"}}},"pull_request":{"url":"https://api.github.com/repos/rewiko/curator/pulls/1","id":163152620,"html_url":"https://github.com/rewiko/curator/pull/1","diff_url":"https://github.com/rewiko/curator/pull/1.diff","patch_url":"https://github.com/rewiko/curator/pull/1.patch","issue_url":"https://api.github.com/repos/rewiko/curator/issues/1","number":1,"state":"open","locked":false,"title":"Fix too big payload 413 Full-Head when we get metadata, create multiple query, one per…","user":{"login":"rewiko","id":1068444,"avatar_url":"https://avatars1.githubusercontent.com/u/1068444?v=4","gravatar_id":"","url":"https://api.github.com/users/rewiko","html_url":"https://github.com/rewiko","followers_url":"https://api.github.com/users/rewiko/followers","following_url":"https://api.github.com/users/rewiko/following{/other_user}","gists_url":"https://api.github.com/users/rewiko/gists{/gist_id}","starred_url":"https://api.github.com/users/rewiko/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rewiko/subscriptions","organizations_url":"https://api.github.com/users/rewiko/orgs","repos_url":"https://api.github.com/users/rewiko/repos","events_url":"https://api.github.com/users/rewiko/events{/privacy}","received_events_url":"https://api.github.com/users/rewiko/received_events","type":"User","site_admin":false},"body":"Fixes Avoid querying all the indices in on go, the payload is too big depending on the number of indeces and the elasticsearch configuration. (specially on AWS when you can't customize the payload parameter) - getting HTTP/1.1 413 FULL head when you have too much indices.\r\n\r\n## Proposed Changes\r\n\r\n  - Making a request per index to get information\r\n\r\n","created_at":"2018-01-16T11:23:31Z","updated_at":"2018-02-09T16:25:12Z","closed_at":null,"merged_at":null,"merge_commit_sha":"e01e262c0fe4c84fb0ca8ce10295a9ce77a2945b","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/rewiko/curator/pulls/1/commits","review_comments_url":"https://api.github.com/repos/rewiko/curator/pulls/1/comments","review_comment_url":"https://api.github.com/repos/rewiko/curator/pulls/comments{/number}","comments_url":"https://api.github.com/repos/rewiko/curator/issues/1/comments","statuses_url":"https://api.github.com/repos/rewiko/curator/statuses/25117d3cc1fc2803d31e1992c4857bf01cee263b","head":{"label":"rewiko:fix-too-big-payload-413-Full-Head","ref":"fix-too-big-payload-413-Full-Head","sha":"25117d3cc1fc2803d31e1992c4857bf01cee263b","user":{"login":"rewiko","id":1068444,"avatar_url":"https://avatars1.githubusercontent.com/u/1068444?v=4","gravatar_id":"","url":"https://api.github.com/users/rewiko","html_url":"https://github.com/rewiko","followers_url":"https://api.github.com/users/rewiko/followers","following_url":"https://api.github.com/users/rewiko/following{/other_user}","gists_url":"https://api.github.com/users/rewiko/gists{/gist_id}","starred_url":"https://api.github.com/users/rewiko/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rewiko/subscriptions","organizations_url":"https://api.github.com/users/rewiko/orgs","repos_url":"https://api.github.com/users/rewiko/repos","events_url":"https://api.github.com/users/rewiko/events{/privacy}","received_events_url":"https://api.github.com/users/rewiko/received_events","type":"User","site_admin":false},"repo":{"id":117673768,"name":"curator","full_name":"rewiko/curator","owner":{"login":"rewiko","id":1068444,"avatar_url":"https://avatars1.githubusercontent.com/u/1068444?v=4","gravatar_id":"","url":"https://api.github.com/users/rewiko","html_url":"https://github.com/rewiko","followers_url":"https://api.github.com/users/rewiko/followers","following_url":"https://api.github.com/users/rewiko/following{/other_user}","gists_url":"https://api.github.com/users/rewiko/gists{/gist_id}","starred_url":"https://api.github.com/users/rewiko/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rewiko/subscriptions","organizations_url":"https://api.github.com/users/rewiko/orgs","repos_url":"https://api.github.com/users/rewiko/repos","events_url":"https://api.github.com/users/rewiko/events{/privacy}","received_events_url":"https://api.github.com/users/rewiko/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/rewiko/curator","description":"Curator: Tending your Elasticsearch indices","fork":true,"url":"https://api.github.com/repos/rewiko/curator","forks_url":"https://api.github.com/repos/rewiko/curator/forks","keys_url":"https://api.github.com/repos/rewiko/curator/keys{/key_id}","collaborators_url":"https://api.github.com/repos/rewiko/curator/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/rewiko/curator/teams","hooks_url":"https://api.github.com/repos/rewiko/curator/hooks","issue_events_url":"https://api.github.com/repos/rewiko/curator/issues/events{/number}","events_url":"https://api.github.com/repos/rewiko/curator/events","assignees_url":"https://api.github.com/repos/rewiko/curator/assignees{/user}","branches_url":"https://api.github.com/repos/rewiko/curator/branches{/branch}","tags_url":"https://api.github.com/repos/rewiko/curator/tags","blobs_url":"https://api.github.com/repos/rewiko/curator/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/rewiko/curator/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/rewiko/curator/git/refs{/sha}","trees_url":"https://api.github.com/repos/rewiko/curator/git/trees{/sha}","statuses_url":"https://api.github.com/repos/rewiko/curator/statuses/{sha}","languages_url":"https://api.github.com/repos/rewiko/curator/languages","stargazers_url":"https://api.github.com/repos/rewiko/curator/stargazers","contributors_url":"https://api.github.com/repos/rewiko/curator/contributors","subscribers_url":"https://api.github.com/repos/rewiko/curator/subscribers","subscription_url":"https://api.github.com/repos/rewiko/curator/subscription","commits_url":"https://api.github.com/repos/rewiko/curator/commits{/sha}","git_commits_url":"https://api.github.com/repos/rewiko/curator/git/commits{/sha}","comments_url":"https://api.github.com/repos/rewiko/curator/comments{/number}","issue_comment_url":"https://api.github.com/repos/rewiko/curator/issues/comments{/number}","contents_url":"https://api.github.com/repos/rewiko/curator/contents/{+path}","compare_url":"https://api.github.com/repos/rewiko/curator/compare/{base}...{head}","merges_url":"https://api.github.com/repos/rewiko/curator/merges","archive_url":"https://api.github.com/repos/rewiko/curator/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/rewiko/curator/downloads","issues_url":"https://api.github.com/repos/rewiko/curator/issues{/number}","pulls_url":"https://api.github.com/repos/rewiko/curator/pulls{/number}","milestones_url":"https://api.github.com/repos/rewiko/curator/milestones{/number}","notifications_url":"https://api.github.com/repos/rewiko/curator/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/rewiko/curator/labels{/name}","releases_url":"https://api.github.com/repos/rewiko/curator/releases{/id}","deployments_url":"https://api.github.com/repos/rewiko/curator/deployments","created_at":"2018-01-16T11:13:16Z","updated_at":"2018-01-16T11:13:18Z","pushed_at":"2018-02-09T13:41:11Z","git_url":"git://github.com/rewiko/curator.git","ssh_url":"git@github.com:rewiko/curator.git","clone_url":"https://github.com/rewiko/curator.git","svn_url":"https://github.com/rewiko/curator","homepage":"","size":2467,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":1,"license":null,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"rewiko:5.x","ref":"5.x","sha":"c76f449f54158fc6a0f62154655903b9ec745bbd","user":{"login":"rewiko","id":1068444,"avatar_url":"https://avatars1.githubusercontent.com/u/1068444?v=4","gravatar_id":"","url":"https://api.github.com/users/rewiko","html_url":"https://github.com/rewiko","followers_url":"https://api.github.com/users/rewiko/followers","following_url":"https://api.github.com/users/rewiko/following{/other_user}","gists_url":"https://api.github.com/users/rewiko/gists{/gist_id}","starred_url":"https://api.github.com/users/rewiko/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rewiko/subscriptions","organizations_url":"https://api.github.com/users/rewiko/orgs","repos_url":"https://api.github.com/users/rewiko/repos","events_url":"https://api.github.com/users/rewiko/events{/privacy}","received_events_url":"https://api.github.com/users/rewiko/received_events","type":"User","site_admin":false},"repo":{"id":117673768,"name":"curator","full_name":"rewiko/curator","owner":{"login":"rewiko","id":1068444,"avatar_url":"https://avatars1.githubusercontent.com/u/1068444?v=4","gravatar_id":"","url":"https://api.github.com/users/rewiko","html_url":"https://github.com/rewiko","followers_url":"https://api.github.com/users/rewiko/followers","following_url":"https://api.github.com/users/rewiko/following{/other_user}","gists_url":"https://api.github.com/users/rewiko/gists{/gist_id}","starred_url":"https://api.github.com/users/rewiko/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rewiko/subscriptions","organizations_url":"https://api.github.com/users/rewiko/orgs","repos_url":"https://api.github.com/users/rewiko/repos","events_url":"https://api.github.com/users/rewiko/events{/privacy}","received_events_url":"https://api.github.com/users/rewiko/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/rewiko/curator","description":"Curator: Tending your Elasticsearch indices","fork":true,"url":"https://api.github.com/repos/rewiko/curator","forks_url":"https://api.github.com/repos/rewiko/curator/forks","keys_url":"https://api.github.com/repos/rewiko/curator/keys{/key_id}","collaborators_url":"https://api.github.com/repos/rewiko/curator/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/rewiko/curator/teams","hooks_url":"https://api.github.com/repos/rewiko/curator/hooks","issue_events_url":"https://api.github.com/repos/rewiko/curator/issues/events{/number}","events_url":"https://api.github.com/repos/rewiko/curator/events","assignees_url":"https://api.github.com/repos/rewiko/curator/assignees{/user}","branches_url":"https://api.github.com/repos/rewiko/curator/branches{/branch}","tags_url":"https://api.github.com/repos/rewiko/curator/tags","blobs_url":"https://api.github.com/repos/rewiko/curator/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/rewiko/curator/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/rewiko/curator/git/refs{/sha}","trees_url":"https://api.github.com/repos/rewiko/curator/git/trees{/sha}","statuses_url":"https://api.github.com/repos/rewiko/curator/statuses/{sha}","languages_url":"https://api.github.com/repos/rewiko/curator/languages","stargazers_url":"https://api.github.com/repos/rewiko/curator/stargazers","contributors_url":"https://api.github.com/repos/rewiko/curator/contributors","subscribers_url":"https://api.github.com/repos/rewiko/curator/subscribers","subscription_url":"https://api.github.com/repos/rewiko/curator/subscription","commits_url":"https://api.github.com/repos/rewiko/curator/commits{/sha}","git_commits_url":"https://api.github.com/repos/rewiko/curator/git/commits{/sha}","comments_url":"https://api.github.com/repos/rewiko/curator/comments{/number}","issue_comment_url":"https://api.github.com/repos/rewiko/curator/issues/comments{/number}","contents_url":"https://api.github.com/repos/rewiko/curator/contents/{+path}","compare_url":"https://api.github.com/repos/rewiko/curator/compare/{base}...{head}","merges_url":"https://api.github.com/repos/rewiko/curator/merges","archive_url":"https://api.github.com/repos/rewiko/curator/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/rewiko/curator/downloads","issues_url":"https://api.github.com/repos/rewiko/curator/issues{/number}","pulls_url":"https://api.github.com/repos/rewiko/curator/pulls{/number}","milestones_url":"https://api.github.com/repos/rewiko/curator/milestones{/number}","notifications_url":"https://api.github.com/repos/rewiko/curator/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/rewiko/curator/labels{/name}","releases_url":"https://api.github.com/repos/rewiko/curator/releases{/id}","deployments_url":"https://api.github.com/repos/rewiko/curator/deployments","created_at":"2018-01-16T11:13:16Z","updated_at":"2018-01-16T11:13:18Z","pushed_at":"2018-02-09T13:41:11Z","git_url":"git://github.com/rewiko/curator.git","ssh_url":"git@github.com:rewiko/curator.git","clone_url":"https://github.com/rewiko/curator.git","svn_url":"https://github.com/rewiko/curator","homepage":"","size":2467,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":1,"license":null,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/rewiko/curator/pulls/1"},"html":{"href":"https://github.com/rewiko/curator/pull/1"},"issue":{"href":"https://api.github.com/repos/rewiko/curator/issues/1"},"comments":{"href":"https://api.github.com/repos/rewiko/curator/issues/1/comments"},"review_comments":{"href":"https://api.github.com/repos/rewiko/curator/pulls/1/comments"},"review_comment":{"href":"https://api.github.com/repos/rewiko/curator/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/rewiko/curator/pulls/1/commits"},"statuses":{"href":"https://api.github.com/repos/rewiko/curator/statuses/25117d3cc1fc2803d31e1992c4857bf01cee263b"}},"author_association":"OWNER"}}
{"actor":{"display_login":"MalsR"}}