{"action":"created","comment":{"url":"https://api.github.com/repos/vertexproject/synapse/pulls/comments/172627824","pull_request_review_id":101677751,"id":172627824,"diff_hunk":"@@ -0,0 +1,443 @@\n+import struct\n+import itertools\n+\n+import lmdb  # used for type resolution\n+\n+import synapse.lib.msgpack as s_msgpack\n+\n+STOR_FLAG_NOINDEX = 0x0001      # there is no byprop index for this prop\n+STOR_FLAG_MULTIVAL = 0x0002     # this is a multi-value prop\n+STOR_FLAG_DEFVAL = 0x0004       # Only set this if it doesn't exist\n+\n+class Seqn:\n+    '''\n+    An append optimized sequence of byte blobs.\n+\n+    Args:\n+        lenv (lmdb.Environment): The LMDB Environment.\n+        name (str): The name of the sequence.\n+    '''\n+    def __init__(self, lenv, name):\n+\n+        self.lenv = lenv\n+        self.db = lenv.open_db(name)\n+\n+        with lenv.begin() as xact:\n+\n+            indx = self.nextindx(xact)\n+            self.indx = itertools.count(indx)\n+\n+    def save(self, xact, items):\n+        '''\n+        Save a series of items to a sequence.\n+\n+        Args:\n+            xact (lmdb.Transaction): An LMDB write transaction.\n+            items (tuple): The series of items to save into the sequence.\n+\n+        Returns:\n+            None\n+        '''\n+        rows = []\n+        for item in items:\n+            byts = s_msgpack.en(item)\n+            lkey = struct.pack('>Q', next(self.indx))\n+            rows.append((lkey, byts))\n+\n+        with xact.cursor(db=self.db) as curs:\n+            curs.putmulti(rows, append=True)\n+\n+    def nextindx(self, xact):\n+        '''\n+        Determine the next insert offset according to storage.\n+\n+        Args:\n+            xact (lmdb.Transaction): An LMDB transaction.\n+\n+        Returns:\n+            int: The next insert offset.\n+        '''\n+        indx = 0\n+        with xact.cursor(db=self.db) as curs:\n+            if curs.last():\n+                indx = struct.unpack('>Q', curs.key())[0] + 1\n+        return indx\n+\n+    def iter(self, xact, offs):\n+        '''\n+        Iterate over items in a sequence from a given offset.\n+\n+        Args:\n+            xact (lmdb.Transaction): An LMDB transaction.\n+            offs (int): The offset to begin iterating from.\n+\n+        Yields:\n+            (indx, valu): The index and valu of the item.\n+        '''\n+        lkey = struct.pack('>Q', offs)\n+        with xact.cursor(db=self.db) as curs:\n+\n+            if not curs.set_key(lkey):\n+                return\n+\n+            for lkey, lval in curs.iternext():\n+                indx = struct.unpack('>Q', lkey)[0]\n+                valu = s_msgpack.un(lval)\n+                yield indx, valu\n+\n+class Metrics:\n+    '''\n+    A helper for recording metrics about an Environment.\n+\n+    Args:\n+        lenv (lmdb.Environment): The LMDB Environment.\n+        name (str): The name of the metrics instance.\n+    '''\n+    def __init__(self, lenv, name=b'metrics'):\n+\n+        self.lenv = lenv\n+\n+        self._db_history = lenv.open_db(name + b':history')\n+        self._db_current = lenv.open_db(name + b':current')\n+\n+        self.info = {}\n+\n+        with lenv.begin() as xact:\n+\n+            for lkey, lval in xact.cursor(db=self._db_current):\n+\n+                name = lkey.decode('utf8')\n+                valu = struct.unpack('>Q', lval)[0]\n+\n+                self.info[name] = valu\n+\n+            indx = 0\n+\n+            curs = xact.cursor(db=self._db_history)\n+            if curs.last():\n+                indx = struct.unpack('>Q', curs.key())[0]","path":"synapse/lib/lmdb.py","position":118,"original_position":118,"commit_id":"e4875c4eb343ac0f2b05e33e6673b82d047da005","original_commit_id":"752ff93efab6482794e32c023b07ad6c75a950e9","user":{"login":"vEpiphyte","id":26100278,"avatar_url":"https://avatars2.githubusercontent.com/u/26100278?v=4","gravatar_id":"","url":"https://api.github.com/users/vEpiphyte","html_url":"https://github.com/vEpiphyte","followers_url":"https://api.github.com/users/vEpiphyte/followers","following_url":"https://api.github.com/users/vEpiphyte/following{/other_user}","gists_url":"https://api.github.com/users/vEpiphyte/gists{/gist_id}","starred_url":"https://api.github.com/users/vEpiphyte/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vEpiphyte/subscriptions","organizations_url":"https://api.github.com/users/vEpiphyte/orgs","repos_url":"https://api.github.com/users/vEpiphyte/repos","events_url":"https://api.github.com/users/vEpiphyte/events{/privacy}","received_events_url":"https://api.github.com/users/vEpiphyte/received_events","type":"User","site_admin":false},"body":"looking","created_at":"2018-03-06T19:13:32Z","updated_at":"2018-03-06T19:13:32Z","html_url":"https://github.com/vertexproject/synapse/pull/650#discussion_r172627824","pull_request_url":"https://api.github.com/repos/vertexproject/synapse/pulls/650","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/vertexproject/synapse/pulls/comments/172627824"},"html":{"href":"https://github.com/vertexproject/synapse/pull/650#discussion_r172627824"},"pull_request":{"href":"https://api.github.com/repos/vertexproject/synapse/pulls/650"}},"in_reply_to_id":168964331},"pull_request":{"url":"https://api.github.com/repos/vertexproject/synapse/pulls/650","id":168018872,"html_url":"https://github.com/vertexproject/synapse/pull/650","diff_url":"https://github.com/vertexproject/synapse/pull/650.diff","patch_url":"https://github.com/vertexproject/synapse/pull/650.patch","issue_url":"https://api.github.com/repos/vertexproject/synapse/issues/650","number":650,"state":"open","locked":false,"title":"WIP: axon mark IX","user":{"login":"invisig0th","id":2624245,"avatar_url":"https://avatars2.githubusercontent.com/u/2624245?v=4","gravatar_id":"","url":"https://api.github.com/users/invisig0th","html_url":"https://github.com/invisig0th","followers_url":"https://api.github.com/users/invisig0th/followers","following_url":"https://api.github.com/users/invisig0th/following{/other_user}","gists_url":"https://api.github.com/users/invisig0th/gists{/gist_id}","starred_url":"https://api.github.com/users/invisig0th/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/invisig0th/subscriptions","organizations_url":"https://api.github.com/users/invisig0th/orgs","repos_url":"https://api.github.com/users/invisig0th/repos","events_url":"https://api.github.com/users/invisig0th/events{/privacy}","received_events_url":"https://api.github.com/users/invisig0th/received_events","type":"User","site_admin":false},"body":"","created_at":"2018-02-08T16:20:27Z","updated_at":"2018-03-06T19:13:32Z","closed_at":null,"merged_at":null,"merge_commit_sha":"2e7a5c1316af6d8029cfe54d9c8569f44c96e7de","assignee":{"login":"invisig0th","id":2624245,"avatar_url":"https://avatars2.githubusercontent.com/u/2624245?v=4","gravatar_id":"","url":"https://api.github.com/users/invisig0th","html_url":"https://github.com/invisig0th","followers_url":"https://api.github.com/users/invisig0th/followers","following_url":"https://api.github.com/users/invisig0th/following{/other_user}","gists_url":"https://api.github.com/users/invisig0th/gists{/gist_id}","starred_url":"https://api.github.com/users/invisig0th/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/invisig0th/subscriptions","organizations_url":"https://api.github.com/users/invisig0th/orgs","repos_url":"https://api.github.com/users/invisig0th/repos","events_url":"https://api.github.com/users/invisig0th/events{/privacy}","received_events_url":"https://api.github.com/users/invisig0th/received_events","type":"User","site_admin":false},"assignees":[{"login":"invisig0th","id":2624245,"avatar_url":"https://avatars2.githubusercontent.com/u/2624245?v=4","gravatar_id":"","url":"https://api.github.com/users/invisig0th","html_url":"https://github.com/invisig0th","followers_url":"https://api.github.com/users/invisig0th/followers","following_url":"https://api.github.com/users/invisig0th/following{/other_user}","gists_url":"https://api.github.com/users/invisig0th/gists{/gist_id}","starred_url":"https://api.github.com/users/invisig0th/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/invisig0th/subscriptions","organizations_url":"https://api.github.com/users/invisig0th/orgs","repos_url":"https://api.github.com/users/invisig0th/repos","events_url":"https://api.github.com/users/invisig0th/events{/privacy}","received_events_url":"https://api.github.com/users/invisig0th/received_events","type":"User","site_admin":false}],"requested_reviewers":[],"requested_teams":[],"labels":[{"id":222989314,"url":"https://api.github.com/repos/vertexproject/synapse/labels/enhancement","name":"enhancement","color":"0052cc","default":true}],"milestone":{"url":"https://api.github.com/repos/vertexproject/synapse/milestones/39","html_url":"https://github.com/vertexproject/synapse/milestone/39","labels_url":"https://api.github.com/repos/vertexproject/synapse/milestones/39/labels","id":3031471,"number":39,"title":"v0.0.46","description":"","creator":{"login":"vEpiphyte","id":26100278,"avatar_url":"https://avatars2.githubusercontent.com/u/26100278?v=4","gravatar_id":"","url":"https://api.github.com/users/vEpiphyte","html_url":"https://github.com/vEpiphyte","followers_url":"https://api.github.com/users/vEpiphyte/followers","following_url":"https://api.github.com/users/vEpiphyte/following{/other_user}","gists_url":"https://api.github.com/users/vEpiphyte/gists{/gist_id}","starred_url":"https://api.github.com/users/vEpiphyte/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vEpiphyte/subscriptions","organizations_url":"https://api.github.com/users/vEpiphyte/orgs","repos_url":"https://api.github.com/users/vEpiphyte/repos","events_url":"https://api.github.com/users/vEpiphyte/events{/privacy}","received_events_url":"https://api.github.com/users/vEpiphyte/received_events","type":"User","site_admin":false},"open_issues":6,"closed_issues":63,"state":"open","created_at":"2018-01-12T13:46:41Z","updated_at":"2018-03-05T19:54:46Z","due_on":"2018-02-19T08:00:00Z","closed_at":null},"commits_url":"https://api.github.com/repos/vertexproject/synapse/pulls/650/commits","review_comments_url":"https://api.github.com/repos/vertexproject/synapse/pulls/650/comments","review_comment_url":"https://api.github.com/repos/vertexproject/synapse/pulls/comments{/number}","comments_url":"https://api.github.com/repos/vertexproject/synapse/issues/650/comments","statuses_url":"https://api.github.com/repos/vertexproject/synapse/statuses/e4875c4eb343ac0f2b05e33e6673b82d047da005","head":{"label":"vertexproject:visi-axon","ref":"visi-axon","sha":"e4875c4eb343ac0f2b05e33e6673b82d047da005","user":{"login":"vertexproject","id":21201579,"avatar_url":"https://avatars2.githubusercontent.com/u/21201579?v=4","gravatar_id":"","url":"https://api.github.com/users/vertexproject","html_url":"https://github.com/vertexproject","followers_url":"https://api.github.com/users/vertexproject/followers","following_url":"https://api.github.com/users/vertexproject/following{/other_user}","gists_url":"https://api.github.com/users/vertexproject/gists{/gist_id}","starred_url":"https://api.github.com/users/vertexproject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vertexproject/subscriptions","organizations_url":"https://api.github.com/users/vertexproject/orgs","repos_url":"https://api.github.com/users/vertexproject/repos","events_url":"https://api.github.com/users/vertexproject/events{/privacy}","received_events_url":"https://api.github.com/users/vertexproject/received_events","type":"Organization","site_admin":false},"repo":{"id":37228107,"name":"synapse","full_name":"vertexproject/synapse","owner":{"login":"vertexproject","id":21201579,"avatar_url":"https://avatars2.githubusercontent.com/u/21201579?v=4","gravatar_id":"","url":"https://api.github.com/users/vertexproject","html_url":"https://github.com/vertexproject","followers_url":"https://api.github.com/users/vertexproject/followers","following_url":"https://api.github.com/users/vertexproject/following{/other_user}","gists_url":"https://api.github.com/users/vertexproject/gists{/gist_id}","starred_url":"https://api.github.com/users/vertexproject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vertexproject/subscriptions","organizations_url":"https://api.github.com/users/vertexproject/orgs","repos_url":"https://api.github.com/users/vertexproject/repos","events_url":"https://api.github.com/users/vertexproject/events{/privacy}","received_events_url":"https://api.github.com/users/vertexproject/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/vertexproject/synapse","description":"Distributed Key-Value Hypergraph Analysis Framework","fork":false,"url":"https://api.github.com/repos/vertexproject/synapse","forks_url":"https://api.github.com/repos/vertexproject/synapse/forks","keys_url":"https://api.github.com/repos/vertexproject/synapse/keys{/key_id}","collaborators_url":"https://api.github.com/repos/vertexproject/synapse/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/vertexproject/synapse/teams","hooks_url":"https://api.github.com/repos/vertexproject/synapse/hooks","issue_events_url":"https://api.github.com/repos/vertexproject/synapse/issues/events{/number}","events_url":"https://api.github.com/repos/vertexproject/synapse/events","assignees_url":"https://api.github.com/repos/vertexproject/synapse/assignees{/user}","branches_url":"https://api.github.com/repos/vertexproject/synapse/branches{/branch}","tags_url":"https://api.github.com/repos/vertexproject/synapse/tags","blobs_url":"https://api.github.com/repos/vertexproject/synapse/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/vertexproject/synapse/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/vertexproject/synapse/git/refs{/sha}","trees_url":"https://api.github.com/repos/vertexproject/synapse/git/trees{/sha}","statuses_url":"https://api.github.com/repos/vertexproject/synapse/statuses/{sha}","languages_url":"https://api.github.com/repos/vertexproject/synapse/languages","stargazers_url":"https://api.github.com/repos/vertexproject/synapse/stargazers","contributors_url":"https://api.github.com/repos/vertexproject/synapse/contributors","subscribers_url":"https://api.github.com/repos/vertexproject/synapse/subscribers","subscription_url":"https://api.github.com/repos/vertexproject/synapse/subscription","commits_url":"https://api.github.com/repos/vertexproject/synapse/commits{/sha}","git_commits_url":"https://api.github.com/repos/vertexproject/synapse/git/commits{/sha}","comments_url":"https://api.github.com/repos/vertexproject/synapse/comments{/number}","issue_comment_url":"https://api.github.com/repos/vertexproject/synapse/issues/comments{/number}","contents_url":"https://api.github.com/repos/vertexproject/synapse/contents/{+path}","compare_url":"https://api.github.com/repos/vertexproject/synapse/compare/{base}...{head}","merges_url":"https://api.github.com/repos/vertexproject/synapse/merges","archive_url":"https://api.github.com/repos/vertexproject/synapse/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/vertexproject/synapse/downloads","issues_url":"https://api.github.com/repos/vertexproject/synapse/issues{/number}","pulls_url":"https://api.github.com/repos/vertexproject/synapse/pulls{/number}","milestones_url":"https://api.github.com/repos/vertexproject/synapse/milestones{/number}","notifications_url":"https://api.github.com/repos/vertexproject/synapse/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/vertexproject/synapse/labels{/name}","releases_url":"https://api.github.com/repos/vertexproject/synapse/releases{/id}","deployments_url":"https://api.github.com/repos/vertexproject/synapse/deployments","created_at":"2015-06-10T23:29:41Z","updated_at":"2018-02-22T17:55:56Z","pushed_at":"2018-03-06T13:33:55Z","git_url":"git://github.com/vertexproject/synapse.git","ssh_url":"git@github.com:vertexproject/synapse.git","clone_url":"https://github.com/vertexproject/synapse.git","svn_url":"https://github.com/vertexproject/synapse","homepage":"","size":6710,"stargazers_count":39,"watchers_count":39,"language":"Python","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":27,"mirror_url":null,"archived":false,"open_issues_count":64,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":27,"open_issues":64,"watchers":39,"default_branch":"master"}},"base":{"label":"vertexproject:master","ref":"master","sha":"574fff036593c8a931553c91d9d1d13fc715d1bd","user":{"login":"vertexproject","id":21201579,"avatar_url":"https://avatars2.githubusercontent.com/u/21201579?v=4","gravatar_id":"","url":"https://api.github.com/users/vertexproject","html_url":"https://github.com/vertexproject","followers_url":"https://api.github.com/users/vertexproject/followers","following_url":"https://api.github.com/users/vertexproject/following{/other_user}","gists_url":"https://api.github.com/users/vertexproject/gists{/gist_id}","starred_url":"https://api.github.com/users/vertexproject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vertexproject/subscriptions","organizations_url":"https://api.github.com/users/vertexproject/orgs","repos_url":"https://api.github.com/users/vertexproject/repos","events_url":"https://api.github.com/users/vertexproject/events{/privacy}","received_events_url":"https://api.github.com/users/vertexproject/received_events","type":"Organization","site_admin":false},"repo":{"id":37228107,"name":"synapse","full_name":"vertexproject/synapse","owner":{"login":"vertexproject","id":21201579,"avatar_url":"https://avatars2.githubusercontent.com/u/21201579?v=4","gravatar_id":"","url":"https://api.github.com/users/vertexproject","html_url":"https://github.com/vertexproject","followers_url":"https://api.github.com/users/vertexproject/followers","following_url":"https://api.github.com/users/vertexproject/following{/other_user}","gists_url":"https://api.github.com/users/vertexproject/gists{/gist_id}","starred_url":"https://api.github.com/users/vertexproject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vertexproject/subscriptions","organizations_url":"https://api.github.com/users/vertexproject/orgs","repos_url":"https://api.github.com/users/vertexproject/repos","events_url":"https://api.github.com/users/vertexproject/events{/privacy}","received_events_url":"https://api.github.com/users/vertexproject/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/vertexproject/synapse","description":"Distributed Key-Value Hypergraph Analysis Framework","fork":false,"url":"https://api.github.com/repos/vertexproject/synapse","forks_url":"https://api.github.com/repos/vertexproject/synapse/forks","keys_url":"https://api.github.com/repos/vertexproject/synapse/keys{/key_id}","collaborators_url":"https://api.github.com/repos/vertexproject/synapse/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/vertexproject/synapse/teams","hooks_url":"https://api.github.com/repos/vertexproject/synapse/hooks","issue_events_url":"https://api.github.com/repos/vertexproject/synapse/issues/events{/number}","events_url":"https://api.github.com/repos/vertexproject/synapse/events","assignees_url":"https://api.github.com/repos/vertexproject/synapse/assignees{/user}","branches_url":"https://api.github.com/repos/vertexproject/synapse/branches{/branch}","tags_url":"https://api.github.com/repos/vertexproject/synapse/tags","blobs_url":"https://api.github.com/repos/vertexproject/synapse/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/vertexproject/synapse/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/vertexproject/synapse/git/refs{/sha}","trees_url":"https://api.github.com/repos/vertexproject/synapse/git/trees{/sha}","statuses_url":"https://api.github.com/repos/vertexproject/synapse/statuses/{sha}","languages_url":"https://api.github.com/repos/vertexproject/synapse/languages","stargazers_url":"https://api.github.com/repos/vertexproject/synapse/stargazers","contributors_url":"https://api.github.com/repos/vertexproject/synapse/contributors","subscribers_url":"https://api.github.com/repos/vertexproject/synapse/subscribers","subscription_url":"https://api.github.com/repos/vertexproject/synapse/subscription","commits_url":"https://api.github.com/repos/vertexproject/synapse/commits{/sha}","git_commits_url":"https://api.github.com/repos/vertexproject/synapse/git/commits{/sha}","comments_url":"https://api.github.com/repos/vertexproject/synapse/comments{/number}","issue_comment_url":"https://api.github.com/repos/vertexproject/synapse/issues/comments{/number}","contents_url":"https://api.github.com/repos/vertexproject/synapse/contents/{+path}","compare_url":"https://api.github.com/repos/vertexproject/synapse/compare/{base}...{head}","merges_url":"https://api.github.com/repos/vertexproject/synapse/merges","archive_url":"https://api.github.com/repos/vertexproject/synapse/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/vertexproject/synapse/downloads","issues_url":"https://api.github.com/repos/vertexproject/synapse/issues{/number}","pulls_url":"https://api.github.com/repos/vertexproject/synapse/pulls{/number}","milestones_url":"https://api.github.com/repos/vertexproject/synapse/milestones{/number}","notifications_url":"https://api.github.com/repos/vertexproject/synapse/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/vertexproject/synapse/labels{/name}","releases_url":"https://api.github.com/repos/vertexproject/synapse/releases{/id}","deployments_url":"https://api.github.com/repos/vertexproject/synapse/deployments","created_at":"2015-06-10T23:29:41Z","updated_at":"2018-02-22T17:55:56Z","pushed_at":"2018-03-06T13:33:55Z","git_url":"git://github.com/vertexproject/synapse.git","ssh_url":"git@github.com:vertexproject/synapse.git","clone_url":"https://github.com/vertexproject/synapse.git","svn_url":"https://github.com/vertexproject/synapse","homepage":"","size":6710,"stargazers_count":39,"watchers_count":39,"language":"Python","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":27,"mirror_url":null,"archived":false,"open_issues_count":64,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":27,"open_issues":64,"watchers":39,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/vertexproject/synapse/pulls/650"},"html":{"href":"https://github.com/vertexproject/synapse/pull/650"},"issue":{"href":"https://api.github.com/repos/vertexproject/synapse/issues/650"},"comments":{"href":"https://api.github.com/repos/vertexproject/synapse/issues/650/comments"},"review_comments":{"href":"https://api.github.com/repos/vertexproject/synapse/pulls/650/comments"},"review_comment":{"href":"https://api.github.com/repos/vertexproject/synapse/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/vertexproject/synapse/pulls/650/commits"},"statuses":{"href":"https://api.github.com/repos/vertexproject/synapse/statuses/e4875c4eb343ac0f2b05e33e6673b82d047da005"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"vEpiphyte"}}