{"action":"created","comment":{"url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/pulls/comments/83506152","pull_request_review_id":4346409,"id":83506152,"diff_hunk":"@@ -137,49 +148,96 @@ cancel_replication(RepId, #user_ctx{name = Name, roles = Roles}) ->\n      end.\n \n \n--spec stream_terminal_docs_info(binary(), user_doc_cb(), any()) -> any().\n-stream_terminal_docs_info(Db, Cb, UserAcc) ->\n+-spec replication_states() -> [atom()].\n+replication_states() ->\n+    ?REPLICATION_STATES.\n+\n+\n+-spec stream_terminal_docs_info(binary(), user_doc_cb(), any(), [atom()]) -> any().\n+stream_terminal_docs_info(Db, Cb, UserAcc, States) ->\n     DDoc = <<\"_replicator\">>,\n     View = <<\"terminal_states\">>,\n     QueryCb = fun handle_replicator_doc_query/2,\n     Args = #mrargs{view_type = map, reduce = false},\n-    Acc = {Db, Cb, UserAcc},\n+    Acc = {Db, Cb, UserAcc, States},\n     try fabric:query_view(Db, DDoc, View, QueryCb, Acc, Args) of\n-    {ok, {Db, Cb, UserAcc1}} ->\n+    {ok, {Db, Cb, UserAcc1, States}} ->\n         UserAcc1\n     catch error:database_does_not_exist ->\n         UserAcc\n     end.\n \n \n-% Streaming doesn't save memory much here but it is used to provided a similar\n-% API as the terminal docs, which is streaming data from the view\n--spec stream_active_docs_info(user_doc_cb(), any()) -> any().\n-stream_active_docs_info(Cb, UserAcc) ->\n-    {Replies, _BadNodes} = rpc:multicall(couch_replicator_doc_processor, docs, []),\n-    lists:foldl(Cb, UserAcc, lists:append(Replies)).\n+-spec stream_active_docs_info(user_doc_cb(), any(), [atom()]) -> any().\n+stream_active_docs_info(Cb, UserAcc, States) ->\n+    Nodes = lists:usort([node() | nodes()]),\n+    stream_active_docs_info(Nodes, Cb, UserAcc, States).\n+\n+\n+-spec stream_active_docs_info([node()], user_doc_cb(), any(), [atom()]) -> any().\n+stream_active_docs_info([], _Cb, UserAcc, _States) ->\n+    UserAcc;\n+\n+stream_active_docs_info([Node | Nodes], Cb, UserAcc, States) ->\n+    case rpc:call(Node, couch_replicator_doc_processor, docs, [States]) of\n+        {badrpc, Reason} ->\n+            ErrMsg = \"Could not get replicator docs from ~p. Error: ~p\",\n+            couch_log:error(ErrMsg, [Node, Reason]),\n+            stream_active_docs_info(Nodes, Cb, UserAcc, States);\n+        Results ->\n+            UserAcc1 = lists:foldl(Cb, UserAcc, Results),\n+            stream_active_docs_info(Nodes, Cb, UserAcc1, States)\n+    end.\n \n \n -spec handle_replicator_doc_query\n     ({row, [_]} , query_acc()) -> {ok, query_acc()};\n     ({error, any()}, query_acc()) -> {error, any()};\n     ({meta, any()}, query_acc()) -> {ok,  query_acc()};\n     (complete, query_acc()) -> {ok, query_acc()}.\n-handle_replicator_doc_query({row, Props}, {Db, Cb, UserAcc}) ->\n+handle_replicator_doc_query({row, Props}, {Db, Cb, UserAcc, States}) ->\n     DocId = couch_util:get_value(id, Props),\n-    State = couch_util:get_value(key, Props),\n+    DocStateBin = couch_util:get_value(key, Props),\n+    DocState = try\n+        erlang:binary_to_existing_atom(DocStateBin, utf8)\n+    catch\n+        error:badarg ->\n+            unknown\n+    end,\n     StateInfo = couch_util:get_value(value, Props),\n-    EjsonInfo = {[\n-        {doc_id, DocId},\n-        {database, Db},\n-        {id, null},\n-        {state, State},\n-        {info, StateInfo}\n-    ]},\n-    {ok, {Db, Cb, Cb(EjsonInfo, UserAcc)}};\n+    % Set the error_count to 1 if failed. This is mainly for consistency as\n+    % jobs from doc_processor and scheduler will have that value set\n+    ErrorCount = case DocState =:= failed of\n+        true ->\n+            1;\n+        false ->\n+            0\n+    end,\n+    case filter_replicator_doc_query(DocState, States) of\n+        true ->\n+            EjsonInfo = {[\n+                {doc_id, DocId},\n+                {database, Db},\n+                {id, null},\n+                {state, DocState},\n+                {error_count, ErrorCount},\n+                {info, StateInfo}\n+            ]},\n+            {ok, {Db, Cb, Cb(EjsonInfo, UserAcc), States}};\n+        false ->\n+            {ok, {Db, Cb, UserAcc, States}}\n+    end;\n handle_replicator_doc_query({error, Reason}, _Acc) ->\n     {error, Reason};\n handle_replicator_doc_query({meta, _Meta}, Acc) ->\n     {ok, Acc};\n handle_replicator_doc_query(complete, Acc) ->\n     {stop, Acc}.\n+\n+\n+filter_replicator_doc_query(unknown, _States) ->","path":"src/couch_replicator.erl","position":136,"original_position":136,"commit_id":"ff34f3806639208c140a093c030882a7bb7e2c27","original_commit_id":"ff34f3806639208c140a093c030882a7bb7e2c27","user":{"login":"sagelywizard","id":167249,"avatar_url":"https://avatars.githubusercontent.com/u/167249?v=3","gravatar_id":"","url":"https://api.github.com/users/sagelywizard","html_url":"https://github.com/sagelywizard","followers_url":"https://api.github.com/users/sagelywizard/followers","following_url":"https://api.github.com/users/sagelywizard/following{/other_user}","gists_url":"https://api.github.com/users/sagelywizard/gists{/gist_id}","starred_url":"https://api.github.com/users/sagelywizard/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sagelywizard/subscriptions","organizations_url":"https://api.github.com/users/sagelywizard/orgs","repos_url":"https://api.github.com/users/sagelywizard/repos","events_url":"https://api.github.com/users/sagelywizard/events{/privacy}","received_events_url":"https://api.github.com/users/sagelywizard/received_events","type":"User","site_admin":false},"body":"Spec?","created_at":"2016-10-14T21:37:16Z","updated_at":"2016-10-14T21:41:58Z","html_url":"https://github.com/cloudant/couchdb-couch-replicator/pull/83#discussion_r83506152","pull_request_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/pulls/83","_links":{"self":{"href":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/pulls/comments/83506152"},"html":{"href":"https://github.com/cloudant/couchdb-couch-replicator/pull/83#discussion_r83506152"},"pull_request":{"href":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/pulls/83"}}},"pull_request":{"url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/pulls/83","id":89288495,"html_url":"https://github.com/cloudant/couchdb-couch-replicator/pull/83","diff_url":"https://github.com/cloudant/couchdb-couch-replicator/pull/83.diff","patch_url":"https://github.com/cloudant/couchdb-couch-replicator/pull/83.patch","issue_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/issues/83","number":83,"state":"open","locked":false,"title":"63012 scheduler better docs","user":{"login":"nickva","id":211822,"avatar_url":"https://avatars.githubusercontent.com/u/211822?v=3","gravatar_id":"","url":"https://api.github.com/users/nickva","html_url":"https://github.com/nickva","followers_url":"https://api.github.com/users/nickva/followers","following_url":"https://api.github.com/users/nickva/following{/other_user}","gists_url":"https://api.github.com/users/nickva/gists{/gist_id}","starred_url":"https://api.github.com/users/nickva/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nickva/subscriptions","organizations_url":"https://api.github.com/users/nickva/orgs","repos_url":"https://api.github.com/users/nickva/repos","events_url":"https://api.github.com/users/nickva/events{/privacy}","received_events_url":"https://api.github.com/users/nickva/received_events","type":"User","site_admin":false},"body":" Update couch_replicator provide a better docs summary\r\n\r\nImproved state filtering. couch_replicator:docs can take a list of state atoms\r\nand return only jobs which are in those states. An empty states list serves as\r\na wildcard.\r\n\r\nReplications which are scheduled return a summary with their scheduling status.\r\nPreviously they returned only their state as seen from the document processor\r\ncompoenent, thus forcing an end-user to understand that there are two separate\r\nsub-systems involved with their own error states and behavior.\r\n\r\nDocument results are provided in somewhat stable sorted order, suitable for\r\nstreaming and pagination via limit and offset. Doc processor ets tables is\r\nsorted by document id and terminal states documents are sorted by their state.\r\nEven though sorting criteria is different is stays relatively consistent between\r\ncalls.\r\n\r\nAlso to support sorted streaming of data, switched rpc:multicall to rpc:calls\r\nwith a sorted list of nodes.","created_at":"2016-10-13T21:45:18Z","updated_at":"2016-10-14T21:41:58Z","closed_at":null,"merged_at":null,"merge_commit_sha":"4f6b01ddf57979180110f79c1bc2e218a9ad8718","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/pulls/83/commits","review_comments_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/pulls/83/comments","review_comment_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/pulls/comments{/number}","comments_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/issues/83/comments","statuses_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/statuses/ff34f3806639208c140a093c030882a7bb7e2c27","head":{"label":"cloudant:63012-scheduler-better-docs","ref":"63012-scheduler-better-docs","sha":"ff34f3806639208c140a093c030882a7bb7e2c27","user":{"login":"cloudant","id":44642,"avatar_url":"https://avatars.githubusercontent.com/u/44642?v=3","gravatar_id":"","url":"https://api.github.com/users/cloudant","html_url":"https://github.com/cloudant","followers_url":"https://api.github.com/users/cloudant/followers","following_url":"https://api.github.com/users/cloudant/following{/other_user}","gists_url":"https://api.github.com/users/cloudant/gists{/gist_id}","starred_url":"https://api.github.com/users/cloudant/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cloudant/subscriptions","organizations_url":"https://api.github.com/users/cloudant/orgs","repos_url":"https://api.github.com/users/cloudant/repos","events_url":"https://api.github.com/users/cloudant/events{/privacy}","received_events_url":"https://api.github.com/users/cloudant/received_events","type":"Organization","site_admin":false},"repo":{"id":38246096,"name":"couchdb-couch-replicator","full_name":"cloudant/couchdb-couch-replicator","owner":{"login":"cloudant","id":44642,"avatar_url":"https://avatars.githubusercontent.com/u/44642?v=3","gravatar_id":"","url":"https://api.github.com/users/cloudant","html_url":"https://github.com/cloudant","followers_url":"https://api.github.com/users/cloudant/followers","following_url":"https://api.github.com/users/cloudant/following{/other_user}","gists_url":"https://api.github.com/users/cloudant/gists{/gist_id}","starred_url":"https://api.github.com/users/cloudant/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cloudant/subscriptions","organizations_url":"https://api.github.com/users/cloudant/orgs","repos_url":"https://api.github.com/users/cloudant/repos","events_url":"https://api.github.com/users/cloudant/events{/privacy}","received_events_url":"https://api.github.com/users/cloudant/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/cloudant/couchdb-couch-replicator","description":"Mirror of Apache CouchDB","fork":true,"url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator","forks_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/forks","keys_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/keys{/key_id}","collaborators_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/teams","hooks_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/hooks","issue_events_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/issues/events{/number}","events_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/events","assignees_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/assignees{/user}","branches_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/branches{/branch}","tags_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/tags","blobs_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/git/refs{/sha}","trees_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/git/trees{/sha}","statuses_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/statuses/{sha}","languages_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/languages","stargazers_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/stargazers","contributors_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/contributors","subscribers_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/subscribers","subscription_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/subscription","commits_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/commits{/sha}","git_commits_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/git/commits{/sha}","comments_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/comments{/number}","issue_comment_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/issues/comments{/number}","contents_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/contents/{+path}","compare_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/compare/{base}...{head}","merges_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/merges","archive_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/downloads","issues_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/issues{/number}","pulls_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/pulls{/number}","milestones_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/milestones{/number}","notifications_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/labels{/name}","releases_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/releases{/id}","deployments_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/deployments","created_at":"2015-06-29T12:19:20Z","updated_at":"2016-05-13T14:06:45Z","pushed_at":"2016-10-14T19:27:57Z","git_url":"git://github.com/cloudant/couchdb-couch-replicator.git","ssh_url":"git@github.com:cloudant/couchdb-couch-replicator.git","clone_url":"https://github.com/cloudant/couchdb-couch-replicator.git","svn_url":"https://github.com/cloudant/couchdb-couch-replicator","homepage":null,"size":5012,"stargazers_count":0,"watchers_count":0,"language":"Erlang","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":2,"forks":0,"open_issues":2,"watchers":0,"default_branch":"master"}},"base":{"label":"cloudant:63012-scheduler","ref":"63012-scheduler","sha":"08597b0b76c425be9557be9b5b70520f78b7ab30","user":{"login":"cloudant","id":44642,"avatar_url":"https://avatars.githubusercontent.com/u/44642?v=3","gravatar_id":"","url":"https://api.github.com/users/cloudant","html_url":"https://github.com/cloudant","followers_url":"https://api.github.com/users/cloudant/followers","following_url":"https://api.github.com/users/cloudant/following{/other_user}","gists_url":"https://api.github.com/users/cloudant/gists{/gist_id}","starred_url":"https://api.github.com/users/cloudant/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cloudant/subscriptions","organizations_url":"https://api.github.com/users/cloudant/orgs","repos_url":"https://api.github.com/users/cloudant/repos","events_url":"https://api.github.com/users/cloudant/events{/privacy}","received_events_url":"https://api.github.com/users/cloudant/received_events","type":"Organization","site_admin":false},"repo":{"id":38246096,"name":"couchdb-couch-replicator","full_name":"cloudant/couchdb-couch-replicator","owner":{"login":"cloudant","id":44642,"avatar_url":"https://avatars.githubusercontent.com/u/44642?v=3","gravatar_id":"","url":"https://api.github.com/users/cloudant","html_url":"https://github.com/cloudant","followers_url":"https://api.github.com/users/cloudant/followers","following_url":"https://api.github.com/users/cloudant/following{/other_user}","gists_url":"https://api.github.com/users/cloudant/gists{/gist_id}","starred_url":"https://api.github.com/users/cloudant/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cloudant/subscriptions","organizations_url":"https://api.github.com/users/cloudant/orgs","repos_url":"https://api.github.com/users/cloudant/repos","events_url":"https://api.github.com/users/cloudant/events{/privacy}","received_events_url":"https://api.github.com/users/cloudant/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/cloudant/couchdb-couch-replicator","description":"Mirror of Apache CouchDB","fork":true,"url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator","forks_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/forks","keys_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/keys{/key_id}","collaborators_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/teams","hooks_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/hooks","issue_events_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/issues/events{/number}","events_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/events","assignees_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/assignees{/user}","branches_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/branches{/branch}","tags_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/tags","blobs_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/git/refs{/sha}","trees_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/git/trees{/sha}","statuses_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/statuses/{sha}","languages_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/languages","stargazers_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/stargazers","contributors_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/contributors","subscribers_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/subscribers","subscription_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/subscription","commits_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/commits{/sha}","git_commits_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/git/commits{/sha}","comments_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/comments{/number}","issue_comment_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/issues/comments{/number}","contents_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/contents/{+path}","compare_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/compare/{base}...{head}","merges_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/merges","archive_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/downloads","issues_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/issues{/number}","pulls_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/pulls{/number}","milestones_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/milestones{/number}","notifications_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/labels{/name}","releases_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/releases{/id}","deployments_url":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/deployments","created_at":"2015-06-29T12:19:20Z","updated_at":"2016-05-13T14:06:45Z","pushed_at":"2016-10-14T19:27:57Z","git_url":"git://github.com/cloudant/couchdb-couch-replicator.git","ssh_url":"git@github.com:cloudant/couchdb-couch-replicator.git","clone_url":"https://github.com/cloudant/couchdb-couch-replicator.git","svn_url":"https://github.com/cloudant/couchdb-couch-replicator","homepage":null,"size":5012,"stargazers_count":0,"watchers_count":0,"language":"Erlang","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":2,"forks":0,"open_issues":2,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/pulls/83"},"html":{"href":"https://github.com/cloudant/couchdb-couch-replicator/pull/83"},"issue":{"href":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/issues/83"},"comments":{"href":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/issues/83/comments"},"review_comments":{"href":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/pulls/83/comments"},"review_comment":{"href":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/pulls/83/commits"},"statuses":{"href":"https://api.github.com/repos/cloudant/couchdb-couch-replicator/statuses/ff34f3806639208c140a093c030882a7bb7e2c27"}}}}
{"actor":{"display_login":"sagelywizard"}}