{"action":"created","comment":{"url":"https://api.github.com/repos/materialsproject/maggma/pulls/comments/206580727","pull_request_review_id":142016378,"id":206580727,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIwNjU4MDcyNw==","diff_hunk":"@@ -0,0 +1,274 @@\n+\"\"\"\n+Example builders for testing and general use.\n+\"\"\"\n+import traceback\n+from abc import ABCMeta, abstractmethod\n+from datetime import datetime\n+\n+from pydash import identity\n+\n+from maggma.builder import Builder\n+from maggma.utils import confirm_field_index, total_size\n+\n+\n+def source_keys_updated(source, target):\n+    \"\"\"\n+    Utility for incremental building. Gets a list of source.key values.\n+\n+    Get key values for source documents that have been updated with respect to\n+    corresponding target documents.\n+\n+    Called should ensure [(lu_field, -1),(key, 1)] compound index on both source\n+    and target.\n+\n+    \"\"\"\n+    keys_updated = set()  # Handle non-unique keys, e.g. for GroupBuilder.\n+    cursor_source = source.query(\n+        properties=[source.key, source.lu_field],\n+        sort=[(source.lu_field, -1), (source.key, 1)])\n+    cursor_target = target.query(\n+        properties=[target.key, target.lu_field],\n+        sort=[(target.lu_field, -1), (target.key, 1)])\n+    tdoc = next(cursor_target, None)\n+    for sdoc in cursor_source:\n+        if tdoc is None:\n+            keys_updated.add(sdoc[source.key])\n+            continue\n+\n+        if tdoc[target.key] == sdoc[source.key]:\n+            if tdoc[target.lu_field] < source.lu_func[0](sdoc[source.lu_field]):\n+                keys_updated.add(sdoc[source.key])\n+            tdoc = next(cursor_target, None)\n+        else:\n+            keys_updated.add(sdoc[source.key])\n+    return list(keys_updated)\n+\n+\n+def get_criteria(source, target, query=None, incremental=True, logger=None):\n+    \"\"\"Return criteria to pass to `source.query` to get items.\"\"\"\n+    index_checks = [confirm_field_index(target, target.key)]\n+    if incremental:\n+        # Ensure [(lu_field, -1), (key, 1)] index on both source and target\n+        for store in (source, target):\n+            info = store.collection.index_information().values()\n+            index_checks.append(\n+                any(spec == [(store.lu_field, -1), (store.key, 1)]\n+                    for spec in (index['key'] for index in info)))\n+    if not all(index_checks):\n+        index_warning = (\n+            \"Missing one or more important indices on stores. \"\n+            \"Performance for large stores may be severely degraded. \"\n+            \"Ensure indices on target.key and \"\n+            \"[(store.lu_field, -1), (store.key, 1)] \"\n+            \"for each of source and target.\"\n+        )\n+        logger.warning(index_warning)\n+\n+    criteria = {}\n+    if query:\n+        criteria.update(query)\n+    if incremental:\n+        logger.info(\n+            \"incremental mode: finding new/updated source keys\")\n+        keys_updated = source_keys_updated(source, target)\n+        # Merge existing criteria and {source.key: {\"$in\": keys_updated}}.\n+        if \"$and\" in criteria:\n+            criteria[\"$and\"].append({source.key: {\"$in\": keys_updated}})\n+        elif source.key in criteria:\n+            # XXX could go deeper and check for $in, but this is fine.\n+            criteria[\"$and\"] = [\n+                {source.key: criteria[source.key].copy()},\n+                {source.key: {\"$in\": keys_updated}},\n+            ]\n+            del criteria[source.key]\n+        else:\n+            criteria.update({source.key: {\"$in\": keys_updated}})\n+    # Check ratio of criteria size to 16 MB MongoDB document size limit.\n+    # Overestimates ratio via 1000 * 1000 instead of 1024 * 1024.\n+    # If criteria is > 16MB, even cursor.count() will fail with a\n+    # \"DocumentTooLarge: \"command document too large\" error.\n+    if (total_size(criteria) / (16 * 1000 * 1000)) >= 1:\n+        raise RuntimeError(\n+            \"`get_items` query criteria too large. This can happen if \"\n+            \"trying to run incremental=True for the initial build of a \"\n+            \"very large source store, or if `query` is too large. You \"\n+            \"can use maggma.utils.total_size to ensure `query` is smaller \"\n+            \"than 16,000,000 bytes.\")\n+    return criteria\n+\n+\n+class MapBuilder(Builder, metaclass=ABCMeta):\n+    \"\"\"\n+    Apply a unary function to yield a target document for each source document.\n+\n+    Supports incremental building, where a source document gets built only if it\n+    has newer (by lu_field) data than the corresponding (by key) target\n+    document.\n+\n+    \"\"\"\n+    def __init__(self, source, target, query=None, incremental=True, **kwargs):\n+        \"\"\"\n+        Apply a unary function to each source document.\n+\n+        Args:\n+            source (Store): source store\n+            target (Store): target store\n+            query (dict): optional query to filter source store\n+            incremental (bool): whether to use lu_field of source and target\n+                to get only new/updated documents.\n+        \"\"\"\n+        self.source = source\n+        self.target = target\n+        self.incremental = incremental\n+        self.query = query\n+        super().__init__(sources=[source], targets=[target], **kwargs)\n+        self.kwargs = kwargs.copy()\n+        self.kwargs.update(query=query, incremental=incremental)\n+\n+    @staticmethod","path":"maggma/examples/builders.py","position":128,"original_position":128,"commit_id":"0684834396dff026bfabb06453ab5f2d3632374c","original_commit_id":"0684834396dff026bfabb06453ab5f2d3632374c","user":{"login":"shyamd","id":16827130,"node_id":"MDQ6VXNlcjE2ODI3MTMw","avatar_url":"https://avatars1.githubusercontent.com/u/16827130?v=4","gravatar_id":"","url":"https://api.github.com/users/shyamd","html_url":"https://github.com/shyamd","followers_url":"https://api.github.com/users/shyamd/followers","following_url":"https://api.github.com/users/shyamd/following{/other_user}","gists_url":"https://api.github.com/users/shyamd/gists{/gist_id}","starred_url":"https://api.github.com/users/shyamd/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/shyamd/subscriptions","organizations_url":"https://api.github.com/users/shyamd/orgs","repos_url":"https://api.github.com/users/shyamd/repos","events_url":"https://api.github.com/users/shyamd/events{/privacy}","received_events_url":"https://api.github.com/users/shyamd/received_events","type":"User","site_admin":false},"body":"why static? Can this and the following static functions be made into properties instead?","created_at":"2018-07-31T15:43:13Z","updated_at":"2018-07-31T15:48:24Z","html_url":"https://github.com/materialsproject/maggma/pull/30#discussion_r206580727","pull_request_url":"https://api.github.com/repos/materialsproject/maggma/pulls/30","author_association":"COLLABORATOR","_links":{"self":{"href":"https://api.github.com/repos/materialsproject/maggma/pulls/comments/206580727"},"html":{"href":"https://github.com/materialsproject/maggma/pull/30#discussion_r206580727"},"pull_request":{"href":"https://api.github.com/repos/materialsproject/maggma/pulls/30"}}},"pull_request":{"url":"https://api.github.com/repos/materialsproject/maggma/pulls/30","id":202986124,"node_id":"MDExOlB1bGxSZXF1ZXN0MjAyOTg2MTI0","html_url":"https://github.com/materialsproject/maggma/pull/30","diff_url":"https://github.com/materialsproject/maggma/pull/30.diff","patch_url":"https://github.com/materialsproject/maggma/pull/30.patch","issue_url":"https://api.github.com/repos/materialsproject/maggma/issues/30","number":30,"state":"open","locked":false,"title":"[WIP] Improve/refactor examples and move inside maggma namespace","user":{"login":"dwinston","id":1497854,"node_id":"MDQ6VXNlcjE0OTc4NTQ=","avatar_url":"https://avatars3.githubusercontent.com/u/1497854?v=4","gravatar_id":"","url":"https://api.github.com/users/dwinston","html_url":"https://github.com/dwinston","followers_url":"https://api.github.com/users/dwinston/followers","following_url":"https://api.github.com/users/dwinston/following{/other_user}","gists_url":"https://api.github.com/users/dwinston/gists{/gist_id}","starred_url":"https://api.github.com/users/dwinston/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dwinston/subscriptions","organizations_url":"https://api.github.com/users/dwinston/orgs","repos_url":"https://api.github.com/users/dwinston/repos","events_url":"https://api.github.com/users/dwinston/events{/privacy}","received_events_url":"https://api.github.com/users/dwinston/received_events","type":"User","site_admin":false},"body":"This is WIP, needs tests, etc.\r\n\r\n* Runner `run()`s without argument so that serialization of runner is complete, i.e. user knows whether it's an MPI runner or not. This also allows one to inspect a runner's processor before the run, e.g. to customize processor log handling. Also, fixes variable-not-existing error in running with MPI.\r\n* Fix `runner_sample`. Give usage info.\r\n* Make a general `MapBuilder` that takes a montyish-serialized unary function as `process_item` to map source documents to target documents by key. Also, improve the robustness of incremental building via a compound sort on last_updated and store key. `CopyBuilder` is now just a `MapBuilder` with `pydash.identity` as `process_item`.\r\n* Various little fixes/refactors.","created_at":"2018-07-21T01:51:03Z","updated_at":"2018-07-31T15:48:24Z","closed_at":null,"merged_at":null,"merge_commit_sha":"84e6aa49b6fba3756f3915f6e715ec913fa4e076","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/materialsproject/maggma/pulls/30/commits","review_comments_url":"https://api.github.com/repos/materialsproject/maggma/pulls/30/comments","review_comment_url":"https://api.github.com/repos/materialsproject/maggma/pulls/comments{/number}","comments_url":"https://api.github.com/repos/materialsproject/maggma/issues/30/comments","statuses_url":"https://api.github.com/repos/materialsproject/maggma/statuses/0684834396dff026bfabb06453ab5f2d3632374c","head":{"label":"materialsproject:improve-examples","ref":"improve-examples","sha":"0684834396dff026bfabb06453ab5f2d3632374c","user":{"login":"materialsproject","id":867057,"node_id":"MDEyOk9yZ2FuaXphdGlvbjg2NzA1Nw==","avatar_url":"https://avatars3.githubusercontent.com/u/867057?v=4","gravatar_id":"","url":"https://api.github.com/users/materialsproject","html_url":"https://github.com/materialsproject","followers_url":"https://api.github.com/users/materialsproject/followers","following_url":"https://api.github.com/users/materialsproject/following{/other_user}","gists_url":"https://api.github.com/users/materialsproject/gists{/gist_id}","starred_url":"https://api.github.com/users/materialsproject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/materialsproject/subscriptions","organizations_url":"https://api.github.com/users/materialsproject/orgs","repos_url":"https://api.github.com/users/materialsproject/repos","events_url":"https://api.github.com/users/materialsproject/events{/privacy}","received_events_url":"https://api.github.com/users/materialsproject/received_events","type":"Organization","site_admin":false},"repo":{"id":94256653,"node_id":"MDEwOlJlcG9zaXRvcnk5NDI1NjY1Mw==","name":"maggma","full_name":"materialsproject/maggma","owner":{"login":"materialsproject","id":867057,"node_id":"MDEyOk9yZ2FuaXphdGlvbjg2NzA1Nw==","avatar_url":"https://avatars3.githubusercontent.com/u/867057?v=4","gravatar_id":"","url":"https://api.github.com/users/materialsproject","html_url":"https://github.com/materialsproject","followers_url":"https://api.github.com/users/materialsproject/followers","following_url":"https://api.github.com/users/materialsproject/following{/other_user}","gists_url":"https://api.github.com/users/materialsproject/gists{/gist_id}","starred_url":"https://api.github.com/users/materialsproject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/materialsproject/subscriptions","organizations_url":"https://api.github.com/users/materialsproject/orgs","repos_url":"https://api.github.com/users/materialsproject/repos","events_url":"https://api.github.com/users/materialsproject/events{/privacy}","received_events_url":"https://api.github.com/users/materialsproject/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/materialsproject/maggma","description":"MongoDB aggregation machine","fork":false,"url":"https://api.github.com/repos/materialsproject/maggma","forks_url":"https://api.github.com/repos/materialsproject/maggma/forks","keys_url":"https://api.github.com/repos/materialsproject/maggma/keys{/key_id}","collaborators_url":"https://api.github.com/repos/materialsproject/maggma/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/materialsproject/maggma/teams","hooks_url":"https://api.github.com/repos/materialsproject/maggma/hooks","issue_events_url":"https://api.github.com/repos/materialsproject/maggma/issues/events{/number}","events_url":"https://api.github.com/repos/materialsproject/maggma/events","assignees_url":"https://api.github.com/repos/materialsproject/maggma/assignees{/user}","branches_url":"https://api.github.com/repos/materialsproject/maggma/branches{/branch}","tags_url":"https://api.github.com/repos/materialsproject/maggma/tags","blobs_url":"https://api.github.com/repos/materialsproject/maggma/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/materialsproject/maggma/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/materialsproject/maggma/git/refs{/sha}","trees_url":"https://api.github.com/repos/materialsproject/maggma/git/trees{/sha}","statuses_url":"https://api.github.com/repos/materialsproject/maggma/statuses/{sha}","languages_url":"https://api.github.com/repos/materialsproject/maggma/languages","stargazers_url":"https://api.github.com/repos/materialsproject/maggma/stargazers","contributors_url":"https://api.github.com/repos/materialsproject/maggma/contributors","subscribers_url":"https://api.github.com/repos/materialsproject/maggma/subscribers","subscription_url":"https://api.github.com/repos/materialsproject/maggma/subscription","commits_url":"https://api.github.com/repos/materialsproject/maggma/commits{/sha}","git_commits_url":"https://api.github.com/repos/materialsproject/maggma/git/commits{/sha}","comments_url":"https://api.github.com/repos/materialsproject/maggma/comments{/number}","issue_comment_url":"https://api.github.com/repos/materialsproject/maggma/issues/comments{/number}","contents_url":"https://api.github.com/repos/materialsproject/maggma/contents/{+path}","compare_url":"https://api.github.com/repos/materialsproject/maggma/compare/{base}...{head}","merges_url":"https://api.github.com/repos/materialsproject/maggma/merges","archive_url":"https://api.github.com/repos/materialsproject/maggma/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/materialsproject/maggma/downloads","issues_url":"https://api.github.com/repos/materialsproject/maggma/issues{/number}","pulls_url":"https://api.github.com/repos/materialsproject/maggma/pulls{/number}","milestones_url":"https://api.github.com/repos/materialsproject/maggma/milestones{/number}","notifications_url":"https://api.github.com/repos/materialsproject/maggma/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/materialsproject/maggma/labels{/name}","releases_url":"https://api.github.com/repos/materialsproject/maggma/releases{/id}","deployments_url":"https://api.github.com/repos/materialsproject/maggma/deployments","created_at":"2017-06-13T20:50:20Z","updated_at":"2018-07-13T18:02:16Z","pushed_at":"2018-07-31T00:30:41Z","git_url":"git://github.com/materialsproject/maggma.git","ssh_url":"git@github.com:materialsproject/maggma.git","clone_url":"https://github.com/materialsproject/maggma.git","svn_url":"https://github.com/materialsproject/maggma","homepage":"","size":5758,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":5,"mirror_url":null,"archived":false,"open_issues_count":5,"license":{"key":"other","name":"Other","spdx_id":null,"url":null,"node_id":"MDc6TGljZW5zZTA="},"forks":5,"open_issues":5,"watchers":1,"default_branch":"master"}},"base":{"label":"materialsproject:master","ref":"master","sha":"c4a4ae5db28d6f3eb9dcb0d745873144fa1bf7c0","user":{"login":"materialsproject","id":867057,"node_id":"MDEyOk9yZ2FuaXphdGlvbjg2NzA1Nw==","avatar_url":"https://avatars3.githubusercontent.com/u/867057?v=4","gravatar_id":"","url":"https://api.github.com/users/materialsproject","html_url":"https://github.com/materialsproject","followers_url":"https://api.github.com/users/materialsproject/followers","following_url":"https://api.github.com/users/materialsproject/following{/other_user}","gists_url":"https://api.github.com/users/materialsproject/gists{/gist_id}","starred_url":"https://api.github.com/users/materialsproject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/materialsproject/subscriptions","organizations_url":"https://api.github.com/users/materialsproject/orgs","repos_url":"https://api.github.com/users/materialsproject/repos","events_url":"https://api.github.com/users/materialsproject/events{/privacy}","received_events_url":"https://api.github.com/users/materialsproject/received_events","type":"Organization","site_admin":false},"repo":{"id":94256653,"node_id":"MDEwOlJlcG9zaXRvcnk5NDI1NjY1Mw==","name":"maggma","full_name":"materialsproject/maggma","owner":{"login":"materialsproject","id":867057,"node_id":"MDEyOk9yZ2FuaXphdGlvbjg2NzA1Nw==","avatar_url":"https://avatars3.githubusercontent.com/u/867057?v=4","gravatar_id":"","url":"https://api.github.com/users/materialsproject","html_url":"https://github.com/materialsproject","followers_url":"https://api.github.com/users/materialsproject/followers","following_url":"https://api.github.com/users/materialsproject/following{/other_user}","gists_url":"https://api.github.com/users/materialsproject/gists{/gist_id}","starred_url":"https://api.github.com/users/materialsproject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/materialsproject/subscriptions","organizations_url":"https://api.github.com/users/materialsproject/orgs","repos_url":"https://api.github.com/users/materialsproject/repos","events_url":"https://api.github.com/users/materialsproject/events{/privacy}","received_events_url":"https://api.github.com/users/materialsproject/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/materialsproject/maggma","description":"MongoDB aggregation machine","fork":false,"url":"https://api.github.com/repos/materialsproject/maggma","forks_url":"https://api.github.com/repos/materialsproject/maggma/forks","keys_url":"https://api.github.com/repos/materialsproject/maggma/keys{/key_id}","collaborators_url":"https://api.github.com/repos/materialsproject/maggma/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/materialsproject/maggma/teams","hooks_url":"https://api.github.com/repos/materialsproject/maggma/hooks","issue_events_url":"https://api.github.com/repos/materialsproject/maggma/issues/events{/number}","events_url":"https://api.github.com/repos/materialsproject/maggma/events","assignees_url":"https://api.github.com/repos/materialsproject/maggma/assignees{/user}","branches_url":"https://api.github.com/repos/materialsproject/maggma/branches{/branch}","tags_url":"https://api.github.com/repos/materialsproject/maggma/tags","blobs_url":"https://api.github.com/repos/materialsproject/maggma/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/materialsproject/maggma/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/materialsproject/maggma/git/refs{/sha}","trees_url":"https://api.github.com/repos/materialsproject/maggma/git/trees{/sha}","statuses_url":"https://api.github.com/repos/materialsproject/maggma/statuses/{sha}","languages_url":"https://api.github.com/repos/materialsproject/maggma/languages","stargazers_url":"https://api.github.com/repos/materialsproject/maggma/stargazers","contributors_url":"https://api.github.com/repos/materialsproject/maggma/contributors","subscribers_url":"https://api.github.com/repos/materialsproject/maggma/subscribers","subscription_url":"https://api.github.com/repos/materialsproject/maggma/subscription","commits_url":"https://api.github.com/repos/materialsproject/maggma/commits{/sha}","git_commits_url":"https://api.github.com/repos/materialsproject/maggma/git/commits{/sha}","comments_url":"https://api.github.com/repos/materialsproject/maggma/comments{/number}","issue_comment_url":"https://api.github.com/repos/materialsproject/maggma/issues/comments{/number}","contents_url":"https://api.github.com/repos/materialsproject/maggma/contents/{+path}","compare_url":"https://api.github.com/repos/materialsproject/maggma/compare/{base}...{head}","merges_url":"https://api.github.com/repos/materialsproject/maggma/merges","archive_url":"https://api.github.com/repos/materialsproject/maggma/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/materialsproject/maggma/downloads","issues_url":"https://api.github.com/repos/materialsproject/maggma/issues{/number}","pulls_url":"https://api.github.com/repos/materialsproject/maggma/pulls{/number}","milestones_url":"https://api.github.com/repos/materialsproject/maggma/milestones{/number}","notifications_url":"https://api.github.com/repos/materialsproject/maggma/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/materialsproject/maggma/labels{/name}","releases_url":"https://api.github.com/repos/materialsproject/maggma/releases{/id}","deployments_url":"https://api.github.com/repos/materialsproject/maggma/deployments","created_at":"2017-06-13T20:50:20Z","updated_at":"2018-07-13T18:02:16Z","pushed_at":"2018-07-31T00:30:41Z","git_url":"git://github.com/materialsproject/maggma.git","ssh_url":"git@github.com:materialsproject/maggma.git","clone_url":"https://github.com/materialsproject/maggma.git","svn_url":"https://github.com/materialsproject/maggma","homepage":"","size":5758,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":5,"mirror_url":null,"archived":false,"open_issues_count":5,"license":{"key":"other","name":"Other","spdx_id":null,"url":null,"node_id":"MDc6TGljZW5zZTA="},"forks":5,"open_issues":5,"watchers":1,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/materialsproject/maggma/pulls/30"},"html":{"href":"https://github.com/materialsproject/maggma/pull/30"},"issue":{"href":"https://api.github.com/repos/materialsproject/maggma/issues/30"},"comments":{"href":"https://api.github.com/repos/materialsproject/maggma/issues/30/comments"},"review_comments":{"href":"https://api.github.com/repos/materialsproject/maggma/pulls/30/comments"},"review_comment":{"href":"https://api.github.com/repos/materialsproject/maggma/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/materialsproject/maggma/pulls/30/commits"},"statuses":{"href":"https://api.github.com/repos/materialsproject/maggma/statuses/0684834396dff026bfabb06453ab5f2d3632374c"}},"author_association":"MEMBER"}}
{"actor":{"display_login":"shyamd"}}