{"action":"created","comment":{"url":"https://api.github.com/repos/FRosner/gitlab-template/pulls/comments/183077925","pull_request_review_id":114011857,"id":183077925,"diff_hunk":"@@ -29,22 +29,54 @@ import de.frosner.gitlabtemplate.Main.{PublicKeyType, Username}\n \n import scala.concurrent.{ExecutionContext, Future}\n \n-class GitlabSource(wsClient: StandaloneWSClient, url: String, privateToken: String, onlyActiveUsers: Boolean)(\n-    implicit ec: ExecutionContext)\n+class GitlabSource(wsClient: StandaloneWSClient,\n+                   url: String,\n+                   privateToken: String,\n+                   onlyActiveUsers: Boolean,\n+                   perPage: Int)(implicit ec: ExecutionContext)\n     extends StrictLogging {\n \n   def getUsers: EitherT[Future, Error, Set[GitlabUser]] = {\n     val activeFilter = if (onlyActiveUsers) \"&active=true\" else \"\"\n-    val request = wsClient\n-      .url(s\"$url/api/v4/users?per_page=100000$activeFilter\")\n-      .withHttpHeaders((\"PRIVATE-TOKEN\", privateToken))\n-    logger.debug(s\"Requesting users: ${request.url}\")\n-    EitherT(request.get().map { response =>\n-      val body = response.body[JsValue]\n-      Json.fromJson[Set[GitlabUser]](body).asEither\n-    }).leftMap(GitlabError)\n+    getUsers(EitherT.right[Future, Error, Set[GitlabUser]](Future.successful(Set())), activeFilter, 1, 0)\n   }\n \n+  private def getUsers(result: EitherT[Future, Error, Set[GitlabUser]],\n+                       activeFilter: String,\n+                       pageNumber: Int,\n+                       prevSize: Int): EitherT[Future, Error, Set[GitlabUser]] =\n+    result.flatMap(\n+      gitlabUsers => {\n+        val currentSize = gitlabUsers.size\n+        if (pageNumber > 1 && prevSize == currentSize) {\n+          // If it is not the first page the results are the same return\n+          EitherT.right(Future.successful(gitlabUsers))\n+        } else { // else recurse\n+          val request = wsClient\n+            .url(s\"$url/api/v4/users?per_page=$perPage&page=$pageNumber$activeFilter\")\n+            .withHttpHeaders((\"PRIVATE-TOKEN\", privateToken))\n+          logger.debug(s\"Requesting users: ${request.url}\")\n+          EitherT(request.get().map { response =>\n+            val body = response.body[JsValue]\n+            Json.fromJson[Set[GitlabUser]](body).asEither\n+          }).leftMap(GitlabError)\n+            .leftMap(_.asInstanceOf[Error])\n+            .flatMap(\n+              newGitlabUsers => {\n+                // FIXME: This might be bad for performance if there are too many pages","path":"src/main/scala/de/frosner/gitlabtemplate/GitlabSource.scala","position":48,"original_position":48,"commit_id":"5062b470a6259a9d6e4b2f78a1e0965600220d04","original_commit_id":"5062b470a6259a9d6e4b2f78a1e0965600220d04","user":{"login":"FRosner","id":3427394,"avatar_url":"https://avatars2.githubusercontent.com/u/3427394?v=4","gravatar_id":"","url":"https://api.github.com/users/FRosner","html_url":"https://github.com/FRosner","followers_url":"https://api.github.com/users/FRosner/followers","following_url":"https://api.github.com/users/FRosner/following{/other_user}","gists_url":"https://api.github.com/users/FRosner/gists{/gist_id}","starred_url":"https://api.github.com/users/FRosner/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/FRosner/subscriptions","organizations_url":"https://api.github.com/users/FRosner/orgs","repos_url":"https://api.github.com/users/FRosner/repos","events_url":"https://api.github.com/users/FRosner/events{/privacy}","received_events_url":"https://api.github.com/users/FRosner/received_events","type":"User","site_admin":false},"body":"Why did you discard the traversing future? I am confused.","created_at":"2018-04-20T14:58:30Z","updated_at":"2018-04-20T14:58:30Z","html_url":"https://github.com/FRosner/gitlab-template/pull/8#discussion_r183077925","pull_request_url":"https://api.github.com/repos/FRosner/gitlab-template/pulls/8","author_association":"OWNER","_links":{"self":{"href":"https://api.github.com/repos/FRosner/gitlab-template/pulls/comments/183077925"},"html":{"href":"https://github.com/FRosner/gitlab-template/pull/8#discussion_r183077925"},"pull_request":{"href":"https://api.github.com/repos/FRosner/gitlab-template/pulls/8"}},"in_reply_to_id":182831467},"pull_request":{"url":"https://api.github.com/repos/FRosner/gitlab-template/pulls/8","id":182839180,"html_url":"https://github.com/FRosner/gitlab-template/pull/8","diff_url":"https://github.com/FRosner/gitlab-template/pull/8.diff","patch_url":"https://github.com/FRosner/gitlab-template/pull/8.patch","issue_url":"https://api.github.com/repos/FRosner/gitlab-template/issues/8","number":8,"state":"open","locked":false,"title":"Added pagination support","user":{"login":"sohaibiftikhar","id":5103613,"avatar_url":"https://avatars3.githubusercontent.com/u/5103613?v=4","gravatar_id":"","url":"https://api.github.com/users/sohaibiftikhar","html_url":"https://github.com/sohaibiftikhar","followers_url":"https://api.github.com/users/sohaibiftikhar/followers","following_url":"https://api.github.com/users/sohaibiftikhar/following{/other_user}","gists_url":"https://api.github.com/users/sohaibiftikhar/gists{/gist_id}","starred_url":"https://api.github.com/users/sohaibiftikhar/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sohaibiftikhar/subscriptions","organizations_url":"https://api.github.com/users/sohaibiftikhar/orgs","repos_url":"https://api.github.com/users/sohaibiftikhar/repos","events_url":"https://api.github.com/users/sohaibiftikhar/events{/privacy}","received_events_url":"https://api.github.com/users/sohaibiftikhar/received_events","type":"User","site_admin":false},"body":"Fixes #6","created_at":"2018-04-19T16:58:28Z","updated_at":"2018-04-20T14:58:30Z","closed_at":null,"merged_at":null,"merge_commit_sha":"25ff8fac293700ae9b9ced6df587af6669ac3cfc","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/FRosner/gitlab-template/pulls/8/commits","review_comments_url":"https://api.github.com/repos/FRosner/gitlab-template/pulls/8/comments","review_comment_url":"https://api.github.com/repos/FRosner/gitlab-template/pulls/comments{/number}","comments_url":"https://api.github.com/repos/FRosner/gitlab-template/issues/8/comments","statuses_url":"https://api.github.com/repos/FRosner/gitlab-template/statuses/5062b470a6259a9d6e4b2f78a1e0965600220d04","head":{"label":"sohaibiftikhar:issue6","ref":"issue6","sha":"5062b470a6259a9d6e4b2f78a1e0965600220d04","user":{"login":"sohaibiftikhar","id":5103613,"avatar_url":"https://avatars3.githubusercontent.com/u/5103613?v=4","gravatar_id":"","url":"https://api.github.com/users/sohaibiftikhar","html_url":"https://github.com/sohaibiftikhar","followers_url":"https://api.github.com/users/sohaibiftikhar/followers","following_url":"https://api.github.com/users/sohaibiftikhar/following{/other_user}","gists_url":"https://api.github.com/users/sohaibiftikhar/gists{/gist_id}","starred_url":"https://api.github.com/users/sohaibiftikhar/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sohaibiftikhar/subscriptions","organizations_url":"https://api.github.com/users/sohaibiftikhar/orgs","repos_url":"https://api.github.com/users/sohaibiftikhar/repos","events_url":"https://api.github.com/users/sohaibiftikhar/events{/privacy}","received_events_url":"https://api.github.com/users/sohaibiftikhar/received_events","type":"User","site_admin":false},"repo":{"id":130246348,"name":"gitlab-template","full_name":"sohaibiftikhar/gitlab-template","owner":{"login":"sohaibiftikhar","id":5103613,"avatar_url":"https://avatars3.githubusercontent.com/u/5103613?v=4","gravatar_id":"","url":"https://api.github.com/users/sohaibiftikhar","html_url":"https://github.com/sohaibiftikhar","followers_url":"https://api.github.com/users/sohaibiftikhar/followers","following_url":"https://api.github.com/users/sohaibiftikhar/following{/other_user}","gists_url":"https://api.github.com/users/sohaibiftikhar/gists{/gist_id}","starred_url":"https://api.github.com/users/sohaibiftikhar/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sohaibiftikhar/subscriptions","organizations_url":"https://api.github.com/users/sohaibiftikhar/orgs","repos_url":"https://api.github.com/users/sohaibiftikhar/repos","events_url":"https://api.github.com/users/sohaibiftikhar/events{/privacy}","received_events_url":"https://api.github.com/users/sohaibiftikhar/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/sohaibiftikhar/gitlab-template","description":"Rendering your authorized_keys files based on Gitlab since 2017!","fork":true,"url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template","forks_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/forks","keys_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/keys{/key_id}","collaborators_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/teams","hooks_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/hooks","issue_events_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/issues/events{/number}","events_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/events","assignees_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/assignees{/user}","branches_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/branches{/branch}","tags_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/tags","blobs_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/git/refs{/sha}","trees_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/git/trees{/sha}","statuses_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/statuses/{sha}","languages_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/languages","stargazers_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/stargazers","contributors_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/contributors","subscribers_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/subscribers","subscription_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/subscription","commits_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/commits{/sha}","git_commits_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/git/commits{/sha}","comments_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/comments{/number}","issue_comment_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/issues/comments{/number}","contents_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/contents/{+path}","compare_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/compare/{base}...{head}","merges_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/merges","archive_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/downloads","issues_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/issues{/number}","pulls_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/pulls{/number}","milestones_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/milestones{/number}","notifications_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/labels{/name}","releases_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/releases{/id}","deployments_url":"https://api.github.com/repos/sohaibiftikhar/gitlab-template/deployments","created_at":"2018-04-19T16:54:27Z","updated_at":"2018-04-19T16:54:30Z","pushed_at":"2018-04-19T17:01:30Z","git_url":"git://github.com/sohaibiftikhar/gitlab-template.git","ssh_url":"git@github.com:sohaibiftikhar/gitlab-template.git","clone_url":"https://github.com/sohaibiftikhar/gitlab-template.git","svn_url":"https://github.com/sohaibiftikhar/gitlab-template","homepage":null,"size":90,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"FRosner:master","ref":"master","sha":"d8643650b9458b19f983d7ea193f1201ae9b4ac0","user":{"login":"FRosner","id":3427394,"avatar_url":"https://avatars2.githubusercontent.com/u/3427394?v=4","gravatar_id":"","url":"https://api.github.com/users/FRosner","html_url":"https://github.com/FRosner","followers_url":"https://api.github.com/users/FRosner/followers","following_url":"https://api.github.com/users/FRosner/following{/other_user}","gists_url":"https://api.github.com/users/FRosner/gists{/gist_id}","starred_url":"https://api.github.com/users/FRosner/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/FRosner/subscriptions","organizations_url":"https://api.github.com/users/FRosner/orgs","repos_url":"https://api.github.com/users/FRosner/repos","events_url":"https://api.github.com/users/FRosner/events{/privacy}","received_events_url":"https://api.github.com/users/FRosner/received_events","type":"User","site_admin":false},"repo":{"id":98277979,"name":"gitlab-template","full_name":"FRosner/gitlab-template","owner":{"login":"FRosner","id":3427394,"avatar_url":"https://avatars2.githubusercontent.com/u/3427394?v=4","gravatar_id":"","url":"https://api.github.com/users/FRosner","html_url":"https://github.com/FRosner","followers_url":"https://api.github.com/users/FRosner/followers","following_url":"https://api.github.com/users/FRosner/following{/other_user}","gists_url":"https://api.github.com/users/FRosner/gists{/gist_id}","starred_url":"https://api.github.com/users/FRosner/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/FRosner/subscriptions","organizations_url":"https://api.github.com/users/FRosner/orgs","repos_url":"https://api.github.com/users/FRosner/repos","events_url":"https://api.github.com/users/FRosner/events{/privacy}","received_events_url":"https://api.github.com/users/FRosner/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/FRosner/gitlab-template","description":"Rendering your authorized_keys files based on Gitlab since 2017!","fork":false,"url":"https://api.github.com/repos/FRosner/gitlab-template","forks_url":"https://api.github.com/repos/FRosner/gitlab-template/forks","keys_url":"https://api.github.com/repos/FRosner/gitlab-template/keys{/key_id}","collaborators_url":"https://api.github.com/repos/FRosner/gitlab-template/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/FRosner/gitlab-template/teams","hooks_url":"https://api.github.com/repos/FRosner/gitlab-template/hooks","issue_events_url":"https://api.github.com/repos/FRosner/gitlab-template/issues/events{/number}","events_url":"https://api.github.com/repos/FRosner/gitlab-template/events","assignees_url":"https://api.github.com/repos/FRosner/gitlab-template/assignees{/user}","branches_url":"https://api.github.com/repos/FRosner/gitlab-template/branches{/branch}","tags_url":"https://api.github.com/repos/FRosner/gitlab-template/tags","blobs_url":"https://api.github.com/repos/FRosner/gitlab-template/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/FRosner/gitlab-template/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/FRosner/gitlab-template/git/refs{/sha}","trees_url":"https://api.github.com/repos/FRosner/gitlab-template/git/trees{/sha}","statuses_url":"https://api.github.com/repos/FRosner/gitlab-template/statuses/{sha}","languages_url":"https://api.github.com/repos/FRosner/gitlab-template/languages","stargazers_url":"https://api.github.com/repos/FRosner/gitlab-template/stargazers","contributors_url":"https://api.github.com/repos/FRosner/gitlab-template/contributors","subscribers_url":"https://api.github.com/repos/FRosner/gitlab-template/subscribers","subscription_url":"https://api.github.com/repos/FRosner/gitlab-template/subscription","commits_url":"https://api.github.com/repos/FRosner/gitlab-template/commits{/sha}","git_commits_url":"https://api.github.com/repos/FRosner/gitlab-template/git/commits{/sha}","comments_url":"https://api.github.com/repos/FRosner/gitlab-template/comments{/number}","issue_comment_url":"https://api.github.com/repos/FRosner/gitlab-template/issues/comments{/number}","contents_url":"https://api.github.com/repos/FRosner/gitlab-template/contents/{+path}","compare_url":"https://api.github.com/repos/FRosner/gitlab-template/compare/{base}...{head}","merges_url":"https://api.github.com/repos/FRosner/gitlab-template/merges","archive_url":"https://api.github.com/repos/FRosner/gitlab-template/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/FRosner/gitlab-template/downloads","issues_url":"https://api.github.com/repos/FRosner/gitlab-template/issues{/number}","pulls_url":"https://api.github.com/repos/FRosner/gitlab-template/pulls{/number}","milestones_url":"https://api.github.com/repos/FRosner/gitlab-template/milestones{/number}","notifications_url":"https://api.github.com/repos/FRosner/gitlab-template/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/FRosner/gitlab-template/labels{/name}","releases_url":"https://api.github.com/repos/FRosner/gitlab-template/releases{/id}","deployments_url":"https://api.github.com/repos/FRosner/gitlab-template/deployments","created_at":"2017-07-25T07:32:31Z","updated_at":"2017-07-25T07:32:50Z","pushed_at":"2018-04-19T17:01:31Z","git_url":"git://github.com/FRosner/gitlab-template.git","ssh_url":"git@github.com:FRosner/gitlab-template.git","clone_url":"https://github.com/FRosner/gitlab-template.git","svn_url":"https://github.com/FRosner/gitlab-template","homepage":null,"size":125,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"archived":false,"open_issues_count":4,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":1,"open_issues":4,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/FRosner/gitlab-template/pulls/8"},"html":{"href":"https://github.com/FRosner/gitlab-template/pull/8"},"issue":{"href":"https://api.github.com/repos/FRosner/gitlab-template/issues/8"},"comments":{"href":"https://api.github.com/repos/FRosner/gitlab-template/issues/8/comments"},"review_comments":{"href":"https://api.github.com/repos/FRosner/gitlab-template/pulls/8/comments"},"review_comment":{"href":"https://api.github.com/repos/FRosner/gitlab-template/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/FRosner/gitlab-template/pulls/8/commits"},"statuses":{"href":"https://api.github.com/repos/FRosner/gitlab-template/statuses/5062b470a6259a9d6e4b2f78a1e0965600220d04"}},"author_association":"NONE"}}
{"actor":{"display_login":"FRosner"}}