{"action":"created","comment":{"url":"https://api.github.com/repos/atlasapi/atlas/pulls/comments/64623529","id":64623529,"diff_hunk":"@@ -0,0 +1,98 @@\n+package org.atlasapi.remotesite.bbc.nitro;\n+\n+import com.google.common.base.Throwables;\n+import com.google.common.collect.ImmutableList;\n+\n+import com.metabroadcast.atlas.glycerin.Glycerin;\n+import com.metabroadcast.atlas.glycerin.GlycerinException;\n+import com.metabroadcast.atlas.glycerin.GlycerinResponse;\n+import com.metabroadcast.atlas.glycerin.model.Programme;\n+import com.metabroadcast.atlas.glycerin.queries.ProgrammesQuery;\n+\n+import java.util.Iterator;\n+import java.util.List;\n+\n+/**\n+ * Used to paginate over Nitro Programmes to reduce the heap overhead.\n+ *\n+ * Takes an iterable of {@link ProgrammesQuery} which is used to get\n+ * the an iterable of Programmes per page.\n+ * Used as part of {@link OffScheduleContentIngestTask}\n+ */\n+public class PaginatedProgrammeRequest implements Iterable<List<Programme>> {\n+\n+    private final Iterable<ProgrammesQuery> programmeQueries;\n+    private final Glycerin client;\n+\n+    public PaginatedProgrammeRequest(Glycerin client, Iterable<ProgrammesQuery> queries) throws GlycerinException {\n+        this.client = client;\n+        this.programmeQueries = queries;\n+    }\n+\n+    @Override\n+    public Iterator<List<Programme>> iterator() {\n+        return new ProgrammeIterator(client, programmeQueries);\n+    }\n+\n+    private static class ProgrammeIterator implements Iterator<List<Programme>> {\n+\n+        private final Glycerin client;\n+        private final Iterator<ProgrammesQuery> programmeQueries;\n+        private GlycerinResponse<Programme> currentResponse;\n+        private Iterable<Programme> currentProgrammes;\n+        private boolean firstPage = false;\n+\n+        public ProgrammeIterator(Glycerin client, Iterable<ProgrammesQuery> programmeQueries) {\n+            this.client = client;\n+            this.programmeQueries = programmeQueries.iterator();\n+        }\n+\n+        /**\n+         * Checks if current page of programmes is present so that next method can be used\n+         * to retrieve the next page of Programmes.\n+         * At first will get the first page as a list, then get next page as a list, if there are any.\n+         * If not will get next Programme object and repeat previous steps.\n+         * This has been done to decrease the heap overhead when querying Nitro.\n+         * @return returns true if the current Programme is present, else returns false.\n+         */\n+        @Override\n+        public boolean hasNext() {\n+            try {\n+                if (currentResponse == null) { // Getting the first page.\n+                    if (programmeQueries.hasNext()) {\n+                        currentResponse = client.execute(programmeQueries.next());\n+                        return true;","path":"src/main/java/org/atlasapi/remotesite/bbc/nitro/PaginatedProgrammeRequest.java","position":64,"original_position":64,"commit_id":"e329c163755d58876e2a18b02eb3783fe8384ed9","original_commit_id":"e329c163755d58876e2a18b02eb3783fe8384ed9","user":{"login":"emilssolmanis","id":663720,"avatar_url":"https://avatars.githubusercontent.com/u/663720?v=3","gravatar_id":"","url":"https://api.github.com/users/emilssolmanis","html_url":"https://github.com/emilssolmanis","followers_url":"https://api.github.com/users/emilssolmanis/followers","following_url":"https://api.github.com/users/emilssolmanis/following{/other_user}","gists_url":"https://api.github.com/users/emilssolmanis/gists{/gist_id}","starred_url":"https://api.github.com/users/emilssolmanis/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/emilssolmanis/subscriptions","organizations_url":"https://api.github.com/users/emilssolmanis/orgs","repos_url":"https://api.github.com/users/emilssolmanis/repos","events_url":"https://api.github.com/users/emilssolmanis/events{/privacy}","received_events_url":"https://api.github.com/users/emilssolmanis/received_events","type":"User","site_admin":false},"body":"This isn't necessarily correct, the response might not have any results. This should instead be `return currentResponse.getResults().hasNext()` or something to that end.","created_at":"2016-05-25T18:09:42Z","updated_at":"2016-05-25T18:09:42Z","html_url":"https://github.com/atlasapi/atlas/pull/961#discussion_r64623529","pull_request_url":"https://api.github.com/repos/atlasapi/atlas/pulls/961","_links":{"self":{"href":"https://api.github.com/repos/atlasapi/atlas/pulls/comments/64623529"},"html":{"href":"https://github.com/atlasapi/atlas/pull/961#discussion_r64623529"},"pull_request":{"href":"https://api.github.com/repos/atlasapi/atlas/pulls/961"}}},"pull_request":{"url":"https://api.github.com/repos/atlasapi/atlas/pulls/961","id":71274263,"html_url":"https://github.com/atlasapi/atlas/pull/961","diff_url":"https://github.com/atlasapi/atlas/pull/961.diff","patch_url":"https://github.com/atlasapi/atlas/pull/961.patch","issue_url":"https://api.github.com/repos/atlasapi/atlas/issues/961","number":961,"state":"open","locked":false,"title":"Adding pagination to decrease the heap overhead size","user":{"login":"MartinsIrbe","id":13367583,"avatar_url":"https://avatars.githubusercontent.com/u/13367583?v=3","gravatar_id":"","url":"https://api.github.com/users/MartinsIrbe","html_url":"https://github.com/MartinsIrbe","followers_url":"https://api.github.com/users/MartinsIrbe/followers","following_url":"https://api.github.com/users/MartinsIrbe/following{/other_user}","gists_url":"https://api.github.com/users/MartinsIrbe/gists{/gist_id}","starred_url":"https://api.github.com/users/MartinsIrbe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/MartinsIrbe/subscriptions","organizations_url":"https://api.github.com/users/MartinsIrbe/orgs","repos_url":"https://api.github.com/users/MartinsIrbe/repos","events_url":"https://api.github.com/users/MartinsIrbe/events{/privacy}","received_events_url":"https://api.github.com/users/MartinsIrbe/received_events","type":"User","site_admin":false},"body":"- Calling Nitro with 30 PIDs at once.\r\n- Paginating over Programmes, page by page (Page size = 30)","created_at":"2016-05-24T18:59:18Z","updated_at":"2016-05-25T18:09:42Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7fcb8132d34e75113f1450baa6f577d2295a5d29","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/atlasapi/atlas/pulls/961/commits","review_comments_url":"https://api.github.com/repos/atlasapi/atlas/pulls/961/comments","review_comment_url":"https://api.github.com/repos/atlasapi/atlas/pulls/comments{/number}","comments_url":"https://api.github.com/repos/atlasapi/atlas/issues/961/comments","statuses_url":"https://api.github.com/repos/atlasapi/atlas/statuses/e329c163755d58876e2a18b02eb3783fe8384ed9","head":{"label":"atlasapi:MBST-15437-calling-nitro-with-30-pids","ref":"MBST-15437-calling-nitro-with-30-pids","sha":"e329c163755d58876e2a18b02eb3783fe8384ed9","user":{"login":"atlasapi","id":325129,"avatar_url":"https://avatars.githubusercontent.com/u/325129?v=3","gravatar_id":"","url":"https://api.github.com/users/atlasapi","html_url":"https://github.com/atlasapi","followers_url":"https://api.github.com/users/atlasapi/followers","following_url":"https://api.github.com/users/atlasapi/following{/other_user}","gists_url":"https://api.github.com/users/atlasapi/gists{/gist_id}","starred_url":"https://api.github.com/users/atlasapi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/atlasapi/subscriptions","organizations_url":"https://api.github.com/users/atlasapi/orgs","repos_url":"https://api.github.com/users/atlasapi/repos","events_url":"https://api.github.com/users/atlasapi/events{/privacy}","received_events_url":"https://api.github.com/users/atlasapi/received_events","type":"Organization","site_admin":false},"repo":{"id":762119,"name":"atlas","full_name":"atlasapi/atlas","owner":{"login":"atlasapi","id":325129,"avatar_url":"https://avatars.githubusercontent.com/u/325129?v=3","gravatar_id":"","url":"https://api.github.com/users/atlasapi","html_url":"https://github.com/atlasapi","followers_url":"https://api.github.com/users/atlasapi/followers","following_url":"https://api.github.com/users/atlasapi/following{/other_user}","gists_url":"https://api.github.com/users/atlasapi/gists{/gist_id}","starred_url":"https://api.github.com/users/atlasapi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/atlasapi/subscriptions","organizations_url":"https://api.github.com/users/atlasapi/orgs","repos_url":"https://api.github.com/users/atlasapi/repos","events_url":"https://api.github.com/users/atlasapi/events{/privacy}","received_events_url":"https://api.github.com/users/atlasapi/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/atlasapi/atlas","description":"the global video & audio index","fork":false,"url":"https://api.github.com/repos/atlasapi/atlas","forks_url":"https://api.github.com/repos/atlasapi/atlas/forks","keys_url":"https://api.github.com/repos/atlasapi/atlas/keys{/key_id}","collaborators_url":"https://api.github.com/repos/atlasapi/atlas/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/atlasapi/atlas/teams","hooks_url":"https://api.github.com/repos/atlasapi/atlas/hooks","issue_events_url":"https://api.github.com/repos/atlasapi/atlas/issues/events{/number}","events_url":"https://api.github.com/repos/atlasapi/atlas/events","assignees_url":"https://api.github.com/repos/atlasapi/atlas/assignees{/user}","branches_url":"https://api.github.com/repos/atlasapi/atlas/branches{/branch}","tags_url":"https://api.github.com/repos/atlasapi/atlas/tags","blobs_url":"https://api.github.com/repos/atlasapi/atlas/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/atlasapi/atlas/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/atlasapi/atlas/git/refs{/sha}","trees_url":"https://api.github.com/repos/atlasapi/atlas/git/trees{/sha}","statuses_url":"https://api.github.com/repos/atlasapi/atlas/statuses/{sha}","languages_url":"https://api.github.com/repos/atlasapi/atlas/languages","stargazers_url":"https://api.github.com/repos/atlasapi/atlas/stargazers","contributors_url":"https://api.github.com/repos/atlasapi/atlas/contributors","subscribers_url":"https://api.github.com/repos/atlasapi/atlas/subscribers","subscription_url":"https://api.github.com/repos/atlasapi/atlas/subscription","commits_url":"https://api.github.com/repos/atlasapi/atlas/commits{/sha}","git_commits_url":"https://api.github.com/repos/atlasapi/atlas/git/commits{/sha}","comments_url":"https://api.github.com/repos/atlasapi/atlas/comments{/number}","issue_comment_url":"https://api.github.com/repos/atlasapi/atlas/issues/comments{/number}","contents_url":"https://api.github.com/repos/atlasapi/atlas/contents/{+path}","compare_url":"https://api.github.com/repos/atlasapi/atlas/compare/{base}...{head}","merges_url":"https://api.github.com/repos/atlasapi/atlas/merges","archive_url":"https://api.github.com/repos/atlasapi/atlas/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/atlasapi/atlas/downloads","issues_url":"https://api.github.com/repos/atlasapi/atlas/issues{/number}","pulls_url":"https://api.github.com/repos/atlasapi/atlas/pulls{/number}","milestones_url":"https://api.github.com/repos/atlasapi/atlas/milestones{/number}","notifications_url":"https://api.github.com/repos/atlasapi/atlas/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/atlasapi/atlas/labels{/name}","releases_url":"https://api.github.com/repos/atlasapi/atlas/releases{/id}","deployments_url":"https://api.github.com/repos/atlasapi/atlas/deployments","created_at":"2010-07-07T16:28:33Z","updated_at":"2016-05-18T08:49:19Z","pushed_at":"2016-05-25T17:49:24Z","git_url":"git://github.com/atlasapi/atlas.git","ssh_url":"git@github.com:atlasapi/atlas.git","clone_url":"https://github.com/atlasapi/atlas.git","svn_url":"https://github.com/atlasapi/atlas","homepage":"http://atlas.metabroadcast.com/","size":24834,"stargazers_count":28,"watchers_count":28,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":11,"mirror_url":null,"open_issues_count":17,"forks":11,"open_issues":17,"watchers":28,"default_branch":"master"}},"base":{"label":"atlasapi:master","ref":"master","sha":"63f18a666b87bc81280ab1561dfb11903790de6a","user":{"login":"atlasapi","id":325129,"avatar_url":"https://avatars.githubusercontent.com/u/325129?v=3","gravatar_id":"","url":"https://api.github.com/users/atlasapi","html_url":"https://github.com/atlasapi","followers_url":"https://api.github.com/users/atlasapi/followers","following_url":"https://api.github.com/users/atlasapi/following{/other_user}","gists_url":"https://api.github.com/users/atlasapi/gists{/gist_id}","starred_url":"https://api.github.com/users/atlasapi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/atlasapi/subscriptions","organizations_url":"https://api.github.com/users/atlasapi/orgs","repos_url":"https://api.github.com/users/atlasapi/repos","events_url":"https://api.github.com/users/atlasapi/events{/privacy}","received_events_url":"https://api.github.com/users/atlasapi/received_events","type":"Organization","site_admin":false},"repo":{"id":762119,"name":"atlas","full_name":"atlasapi/atlas","owner":{"login":"atlasapi","id":325129,"avatar_url":"https://avatars.githubusercontent.com/u/325129?v=3","gravatar_id":"","url":"https://api.github.com/users/atlasapi","html_url":"https://github.com/atlasapi","followers_url":"https://api.github.com/users/atlasapi/followers","following_url":"https://api.github.com/users/atlasapi/following{/other_user}","gists_url":"https://api.github.com/users/atlasapi/gists{/gist_id}","starred_url":"https://api.github.com/users/atlasapi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/atlasapi/subscriptions","organizations_url":"https://api.github.com/users/atlasapi/orgs","repos_url":"https://api.github.com/users/atlasapi/repos","events_url":"https://api.github.com/users/atlasapi/events{/privacy}","received_events_url":"https://api.github.com/users/atlasapi/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/atlasapi/atlas","description":"the global video & audio index","fork":false,"url":"https://api.github.com/repos/atlasapi/atlas","forks_url":"https://api.github.com/repos/atlasapi/atlas/forks","keys_url":"https://api.github.com/repos/atlasapi/atlas/keys{/key_id}","collaborators_url":"https://api.github.com/repos/atlasapi/atlas/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/atlasapi/atlas/teams","hooks_url":"https://api.github.com/repos/atlasapi/atlas/hooks","issue_events_url":"https://api.github.com/repos/atlasapi/atlas/issues/events{/number}","events_url":"https://api.github.com/repos/atlasapi/atlas/events","assignees_url":"https://api.github.com/repos/atlasapi/atlas/assignees{/user}","branches_url":"https://api.github.com/repos/atlasapi/atlas/branches{/branch}","tags_url":"https://api.github.com/repos/atlasapi/atlas/tags","blobs_url":"https://api.github.com/repos/atlasapi/atlas/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/atlasapi/atlas/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/atlasapi/atlas/git/refs{/sha}","trees_url":"https://api.github.com/repos/atlasapi/atlas/git/trees{/sha}","statuses_url":"https://api.github.com/repos/atlasapi/atlas/statuses/{sha}","languages_url":"https://api.github.com/repos/atlasapi/atlas/languages","stargazers_url":"https://api.github.com/repos/atlasapi/atlas/stargazers","contributors_url":"https://api.github.com/repos/atlasapi/atlas/contributors","subscribers_url":"https://api.github.com/repos/atlasapi/atlas/subscribers","subscription_url":"https://api.github.com/repos/atlasapi/atlas/subscription","commits_url":"https://api.github.com/repos/atlasapi/atlas/commits{/sha}","git_commits_url":"https://api.github.com/repos/atlasapi/atlas/git/commits{/sha}","comments_url":"https://api.github.com/repos/atlasapi/atlas/comments{/number}","issue_comment_url":"https://api.github.com/repos/atlasapi/atlas/issues/comments{/number}","contents_url":"https://api.github.com/repos/atlasapi/atlas/contents/{+path}","compare_url":"https://api.github.com/repos/atlasapi/atlas/compare/{base}...{head}","merges_url":"https://api.github.com/repos/atlasapi/atlas/merges","archive_url":"https://api.github.com/repos/atlasapi/atlas/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/atlasapi/atlas/downloads","issues_url":"https://api.github.com/repos/atlasapi/atlas/issues{/number}","pulls_url":"https://api.github.com/repos/atlasapi/atlas/pulls{/number}","milestones_url":"https://api.github.com/repos/atlasapi/atlas/milestones{/number}","notifications_url":"https://api.github.com/repos/atlasapi/atlas/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/atlasapi/atlas/labels{/name}","releases_url":"https://api.github.com/repos/atlasapi/atlas/releases{/id}","deployments_url":"https://api.github.com/repos/atlasapi/atlas/deployments","created_at":"2010-07-07T16:28:33Z","updated_at":"2016-05-18T08:49:19Z","pushed_at":"2016-05-25T17:49:24Z","git_url":"git://github.com/atlasapi/atlas.git","ssh_url":"git@github.com:atlasapi/atlas.git","clone_url":"https://github.com/atlasapi/atlas.git","svn_url":"https://github.com/atlasapi/atlas","homepage":"http://atlas.metabroadcast.com/","size":24834,"stargazers_count":28,"watchers_count":28,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":11,"mirror_url":null,"open_issues_count":17,"forks":11,"open_issues":17,"watchers":28,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/atlasapi/atlas/pulls/961"},"html":{"href":"https://github.com/atlasapi/atlas/pull/961"},"issue":{"href":"https://api.github.com/repos/atlasapi/atlas/issues/961"},"comments":{"href":"https://api.github.com/repos/atlasapi/atlas/issues/961/comments"},"review_comments":{"href":"https://api.github.com/repos/atlasapi/atlas/pulls/961/comments"},"review_comment":{"href":"https://api.github.com/repos/atlasapi/atlas/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/atlasapi/atlas/pulls/961/commits"},"statuses":{"href":"https://api.github.com/repos/atlasapi/atlas/statuses/e329c163755d58876e2a18b02eb3783fe8384ed9"}}}}
{"actor":{"display_login":"emilssolmanis"}}