{"action":"created","comment":{"url":"https://api.github.com/repos/RAttab/rill/pulls/comments/177229898","pull_request_review_id":107067528,"id":177229898,"diff_hunk":"@@ -339,144 +374,255 @@ static void writer_close(\n     close(store->fd);\n }\n \n+static void init_store_offsets(struct rill_store* store, uint64_t pairs)\n+{\n+    store->head->index_b_off = store->head->index_a_off + indexer_cap(pairs);\n+    store->head->data_a_off = store->head->index_b_off + indexer_cap(pairs);\n+\n+    store->index_a = (void *) ((uintptr_t) store->vma + store->head->index_a_off);\n+    store->index_b = (void *) ((uintptr_t) store->vma + store->head->index_b_off);\n+    store->data_a = (void *) ((uintptr_t) store->vma + store->head->data_a_off);\n+}\n+\n+static void prepare_col_b_offsets(\n+    struct rill_store* store, struct encoder* coder_a)\n+{\n+    store->head->data_b_off = store->head->data_a_off + coder_off(coder_a);\n+    store->data_b = (void *) ((uintptr_t) store->vma + store->head->data_b_off);\n+}\n+\n bool rill_store_write(\n         const char *file,\n-        rill_ts_t ts, size_t quant,\n+        rill_ts_t ts,\n+        size_t quant,\n         struct rill_pairs *pairs)\n {\n     rill_pairs_compact(pairs);\n     if (!pairs->len) return true;\n \n     struct vals *vals = vals_from_pairs(pairs);\n     if (!vals) goto fail_vals;\n+    struct vals *invert_vals = keys_from_pairs(pairs); // TODO fix fail cleanup\n \n     struct rill_store store = {0};\n-    if (!writer_open(&store, file, vals, pairs->len, ts, quant)) {\n+    if (!writer_open(&store, file, vals, invert_vals,\n+                     pairs->len, ts, quant)) {\n         rill_fail(\"unable to create '%s'\", file);\n         goto fail_open;\n     }\n \n-    struct indexer *indexer = indexer_alloc(pairs->len);\n-    struct encoder coder = writer_begin(&store, vals, indexer);\n+    struct indexer *indexer_a = indexer_alloc(pairs->len); // TODO catch fail alloc here\n+    if (!indexer_a) goto fail_indexer_a_alloc;\n+    struct indexer *indexer_b = indexer_alloc(pairs->len); // TODO catch fail alloc here\n+    if (!indexer_b) goto fail_indexer_b_alloc;\n+\n+    init_store_offsets(&store, pairs->len);\n+\n+    struct encoder coder_a =\n+        store_encoder(&store, indexer_a, vals, store.head->data_a_off);\n+\n+    for (size_t i = 0; i < pairs->len; ++i) {\n+        if (!coder_encode(&coder_a, &pairs->data[i])) goto fail_encode_a;\n+    }\n+    if (!coder_finish(&coder_a)) goto fail_encode_a;\n+\n+    prepare_col_b_offsets(&store, &coder_a);\n+\n+    struct encoder coder_b =\n+        store_encoder(&store, indexer_b, invert_vals, store.head->data_b_off);\n+\n+    rill_pairs_invert(pairs);\n+    rill_pairs_compact(pairs); /* recompact mainly for sort */\n \n     for (size_t i = 0; i < pairs->len; ++i) {\n-        if (!coder_encode(&coder, &pairs->data[i])) goto fail_encode;\n+        if (!coder_encode(&coder_b, &pairs->data[i])) goto fail_encode_b;\n     }\n-    if (!coder_finish(&coder)) goto fail_encode;\n+    if (!coder_finish(&coder_b)) goto fail_encode_b;\n+\n+    writer_flush_indices(&store, indexer_a, indexer_b);\n+\n+    store.head->pairs = coder_a.pairs;\n \n-    store.head->keys = coder.keys;\n-    store.head->pairs = coder.pairs;\n+    writer_close(&store, store.head->data_b_off + coder_off(&coder_b));\n \n-    writer_close(&store, indexer, store.head->data_off + coder_off(&coder));\n+    coder_close(&coder_a);\n+    coder_close(&coder_b);\n+\n+    indexer_free(indexer_a);\n+    indexer_free(indexer_b);\n \n-    coder_close(&coder);\n-    indexer_free(indexer);\n     free(vals);\n+    free(invert_vals);\n+\n     return true;\n \n-  fail_encode:\n-    coder_close(&coder);\n-    writer_close(&store, indexer, 0);\n-    indexer_free(indexer);\n+  fail_encode_b:\n+    coder_close(&coder_b);\n+    writer_close(&store, 0);\n+  fail_encode_a:\n+    coder_close(&coder_a);\n+    indexer_free(indexer_b);\n+  fail_indexer_b_alloc:\n+    indexer_free(indexer_a);\n+  fail_indexer_a_alloc:\n   fail_open:\n     free(vals);\n   fail_vals:\n     return false;\n }\n \n+static struct vals *vals_from_index(struct index* index)\n+{\n+    struct vals *vals =\n+        calloc(1, sizeof(*vals) + sizeof(vals->data[0]) * index->len);\n+\n+    vals->len = index->len;\n+\n+    for (size_t i = 0; i < index->len; ++i) {\n+        vals->data[i] = index->data[i].key;\n+    }\n+\n+    return vals;\n+}\n+\n+static bool merge_with_config(\n+    struct encoder* coder,\n+    struct rill_store** list,\n+    size_t list_len,\n+    enum rill_col col)\n+{\n+  struct rill_kv kvs[list_len];\n+\n+  struct decoder decoders[list_len];\n+\n+  size_t it_len = 0;\n+  for (size_t i = 0; i < list_len; ++i) {\n+    if (!list[i]) continue;\n+    decoders[it_len] = store_decoder(list[i], col);\n+    it_len++;\n+  }\n+  assert(it_len);\n+\n+  for (size_t i = 0; i < it_len; ++i) {\n+    if (!(coder_decode(&decoders[i], &kvs[i]))) { /* goto fail_coder; // TODO FIXME */ }\n+  }\n+\n+  struct rill_kv prev = {0};\n+\n+  while (it_len > 0) { /* while there are valid entries of rill db files */\n+    size_t target = 0;\n+\n+    for (size_t i = 1; i < it_len; ++i) {\n+      if (rill_kv_cmp(&kvs[i], &kvs[target]) < 0)\n+        target = i;\n+    }\n+\n+    struct rill_kv *kv = &kvs[target];\n+    struct decoder *decoder = &decoders[target];\n+\n+    if (rill_likely(rill_kv_nil(&prev) || rill_kv_cmp(&prev, kv) < 0)) {\n+      if (!coder_encode(coder, kv)) { /* goto fail_coder; // TODO FIXME */ }\n+      prev = *kv;\n+    }\n+\n+    if (!coder_decode(decoder, kv)) { /* goto fail_coder; // TODO FIXME */ }\n+    if (rill_unlikely(rill_kv_nil(kv))) {\n+      memmove(kvs + target,\n+              kvs + target + 1,\n+              (it_len - target - 1) * sizeof(kvs[0]));\n+      memmove(decoders + target,\n+              decoders + target + 1,\n+              (it_len - target - 1) * sizeof(decoders[0]));\n+      it_len--;\n+    }\n+  }\n+\n+  return true;\n+}\n+\n bool rill_store_merge(\n         const char *file,\n         rill_ts_t ts, size_t quant,\n         struct rill_store **list, size_t list_len)\n {\n     assert(list_len > 1);\n \n-    size_t keys = 0;\n     size_t pairs = 0;\n     struct vals *vals = NULL;\n+    struct vals *invert_vals = NULL;\n \n-    struct decoder decoders[list_len];\n-    struct rill_kv kvs[list_len];\n-\n-    size_t it_len = 0;\n     for (size_t i = 0; i < list_len; ++i) {\n         if (!list[i]) continue;\n         vma_will_need(list[i]);\n \n-        struct vals *ret = vals_merge(vals, list[i]->vals);\n-        if (ret) { vals = ret; } else { goto fail_vals; }\n+        struct vals *merge_from_index = vals_from_index(list[i]->index_b);\n+        struct vals *invert_merge_from_index = vals_from_index(list[i]->index_a);\n+        struct vals *ret = vals_merge(vals, merge_from_index);\n+        struct vals *iret = vals_merge(invert_vals, invert_merge_from_index);","path":"src/store.c","position":485,"original_position":470,"commit_id":"4745928f1613ba6403f525bb503bbae2cf2da27e","original_commit_id":"8a875ab33e995140a34cbac0871a06560f16d613","user":{"login":"RAttab","id":652873,"avatar_url":"https://avatars1.githubusercontent.com/u/652873?v=4","gravatar_id":"","url":"https://api.github.com/users/RAttab","html_url":"https://github.com/RAttab","followers_url":"https://api.github.com/users/RAttab/followers","following_url":"https://api.github.com/users/RAttab/following{/other_user}","gists_url":"https://api.github.com/users/RAttab/gists{/gist_id}","starred_url":"https://api.github.com/users/RAttab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/RAttab/subscriptions","organizations_url":"https://api.github.com/users/RAttab/orgs","repos_url":"https://api.github.com/users/RAttab/repos","events_url":"https://api.github.com/users/RAttab/events{/privacy}","received_events_url":"https://api.github.com/users/RAttab/received_events","type":"User","site_admin":false},"body":"Instead of loading the values in a list then merging them in, you could simply merge them in from the index directly. This will greatly simplify the logic and avoid memory leaks on error which I think are there right now.","created_at":"2018-03-26T20:49:36Z","updated_at":"2018-03-26T20:49:36Z","html_url":"https://github.com/RAttab/rill/pull/6#discussion_r177229898","pull_request_url":"https://api.github.com/repos/RAttab/rill/pulls/6","author_association":"OWNER","_links":{"self":{"href":"https://api.github.com/repos/RAttab/rill/pulls/comments/177229898"},"html":{"href":"https://github.com/RAttab/rill/pull/6#discussion_r177229898"},"pull_request":{"href":"https://api.github.com/repos/RAttab/rill/pulls/6"}}},"pull_request":{"url":"https://api.github.com/repos/RAttab/rill/pulls/6","id":170030322,"html_url":"https://github.com/RAttab/rill/pull/6","diff_url":"https://github.com/RAttab/rill/pull/6.diff","patch_url":"https://github.com/RAttab/rill/pull/6.patch","issue_url":"https://api.github.com/repos/RAttab/rill/issues/6","number":6,"state":"open","locked":false,"title":"[wip] Add reverse lookup","user":{"login":"psyomn","id":505688,"avatar_url":"https://avatars1.githubusercontent.com/u/505688?v=4","gravatar_id":"","url":"https://api.github.com/users/psyomn","html_url":"https://github.com/psyomn","followers_url":"https://api.github.com/users/psyomn/followers","following_url":"https://api.github.com/users/psyomn/following{/other_user}","gists_url":"https://api.github.com/users/psyomn/gists{/gist_id}","starred_url":"https://api.github.com/users/psyomn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/psyomn/subscriptions","organizations_url":"https://api.github.com/users/psyomn/orgs","repos_url":"https://api.github.com/users/psyomn/repos","events_url":"https://api.github.com/users/psyomn/events{/privacy}","received_events_url":"https://api.github.com/users/psyomn/received_events","type":"User","site_admin":false},"body":"Note that a lot of the things in here will be cleaned when ready to merge.","created_at":"2018-02-19T18:41:49Z","updated_at":"2018-03-26T20:49:36Z","closed_at":null,"merged_at":null,"merge_commit_sha":"84369a44adcb29c2fc07cf22397aadf2cb50933a","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/RAttab/rill/pulls/6/commits","review_comments_url":"https://api.github.com/repos/RAttab/rill/pulls/6/comments","review_comment_url":"https://api.github.com/repos/RAttab/rill/pulls/comments{/number}","comments_url":"https://api.github.com/repos/RAttab/rill/issues/6/comments","statuses_url":"https://api.github.com/repos/RAttab/rill/statuses/4745928f1613ba6403f525bb503bbae2cf2da27e","head":{"label":"psyomn:feature/the-upside-down","ref":"feature/the-upside-down","sha":"4745928f1613ba6403f525bb503bbae2cf2da27e","user":{"login":"psyomn","id":505688,"avatar_url":"https://avatars1.githubusercontent.com/u/505688?v=4","gravatar_id":"","url":"https://api.github.com/users/psyomn","html_url":"https://github.com/psyomn","followers_url":"https://api.github.com/users/psyomn/followers","following_url":"https://api.github.com/users/psyomn/following{/other_user}","gists_url":"https://api.github.com/users/psyomn/gists{/gist_id}","starred_url":"https://api.github.com/users/psyomn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/psyomn/subscriptions","organizations_url":"https://api.github.com/users/psyomn/orgs","repos_url":"https://api.github.com/users/psyomn/repos","events_url":"https://api.github.com/users/psyomn/events{/privacy}","received_events_url":"https://api.github.com/users/psyomn/received_events","type":"User","site_admin":false},"repo":{"id":106469683,"name":"rill","full_name":"psyomn/rill","owner":{"login":"psyomn","id":505688,"avatar_url":"https://avatars1.githubusercontent.com/u/505688?v=4","gravatar_id":"","url":"https://api.github.com/users/psyomn","html_url":"https://github.com/psyomn","followers_url":"https://api.github.com/users/psyomn/followers","following_url":"https://api.github.com/users/psyomn/following{/other_user}","gists_url":"https://api.github.com/users/psyomn/gists{/gist_id}","starred_url":"https://api.github.com/users/psyomn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/psyomn/subscriptions","organizations_url":"https://api.github.com/users/psyomn/orgs","repos_url":"https://api.github.com/users/psyomn/repos","events_url":"https://api.github.com/users/psyomn/events{/privacy}","received_events_url":"https://api.github.com/users/psyomn/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/psyomn/rill","description":"one-off database","fork":true,"url":"https://api.github.com/repos/psyomn/rill","forks_url":"https://api.github.com/repos/psyomn/rill/forks","keys_url":"https://api.github.com/repos/psyomn/rill/keys{/key_id}","collaborators_url":"https://api.github.com/repos/psyomn/rill/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/psyomn/rill/teams","hooks_url":"https://api.github.com/repos/psyomn/rill/hooks","issue_events_url":"https://api.github.com/repos/psyomn/rill/issues/events{/number}","events_url":"https://api.github.com/repos/psyomn/rill/events","assignees_url":"https://api.github.com/repos/psyomn/rill/assignees{/user}","branches_url":"https://api.github.com/repos/psyomn/rill/branches{/branch}","tags_url":"https://api.github.com/repos/psyomn/rill/tags","blobs_url":"https://api.github.com/repos/psyomn/rill/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/psyomn/rill/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/psyomn/rill/git/refs{/sha}","trees_url":"https://api.github.com/repos/psyomn/rill/git/trees{/sha}","statuses_url":"https://api.github.com/repos/psyomn/rill/statuses/{sha}","languages_url":"https://api.github.com/repos/psyomn/rill/languages","stargazers_url":"https://api.github.com/repos/psyomn/rill/stargazers","contributors_url":"https://api.github.com/repos/psyomn/rill/contributors","subscribers_url":"https://api.github.com/repos/psyomn/rill/subscribers","subscription_url":"https://api.github.com/repos/psyomn/rill/subscription","commits_url":"https://api.github.com/repos/psyomn/rill/commits{/sha}","git_commits_url":"https://api.github.com/repos/psyomn/rill/git/commits{/sha}","comments_url":"https://api.github.com/repos/psyomn/rill/comments{/number}","issue_comment_url":"https://api.github.com/repos/psyomn/rill/issues/comments{/number}","contents_url":"https://api.github.com/repos/psyomn/rill/contents/{+path}","compare_url":"https://api.github.com/repos/psyomn/rill/compare/{base}...{head}","merges_url":"https://api.github.com/repos/psyomn/rill/merges","archive_url":"https://api.github.com/repos/psyomn/rill/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/psyomn/rill/downloads","issues_url":"https://api.github.com/repos/psyomn/rill/issues{/number}","pulls_url":"https://api.github.com/repos/psyomn/rill/pulls{/number}","milestones_url":"https://api.github.com/repos/psyomn/rill/milestones{/number}","notifications_url":"https://api.github.com/repos/psyomn/rill/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/psyomn/rill/labels{/name}","releases_url":"https://api.github.com/repos/psyomn/rill/releases{/id}","deployments_url":"https://api.github.com/repos/psyomn/rill/deployments","created_at":"2017-10-10T20:48:44Z","updated_at":"2017-10-10T20:48:46Z","pushed_at":"2018-03-24T01:46:56Z","git_url":"git://github.com/psyomn/rill.git","ssh_url":"git@github.com:psyomn/rill.git","clone_url":"https://github.com/psyomn/rill.git","svn_url":"https://github.com/psyomn/rill","homepage":null,"size":130,"stargazers_count":0,"watchers_count":0,"language":"C","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":null,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"RAttab:master","ref":"master","sha":"802707bb59815ea71b53c7bb2f08f43a546a7dbf","user":{"login":"RAttab","id":652873,"avatar_url":"https://avatars1.githubusercontent.com/u/652873?v=4","gravatar_id":"","url":"https://api.github.com/users/RAttab","html_url":"https://github.com/RAttab","followers_url":"https://api.github.com/users/RAttab/followers","following_url":"https://api.github.com/users/RAttab/following{/other_user}","gists_url":"https://api.github.com/users/RAttab/gists{/gist_id}","starred_url":"https://api.github.com/users/RAttab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/RAttab/subscriptions","organizations_url":"https://api.github.com/users/RAttab/orgs","repos_url":"https://api.github.com/users/RAttab/repos","events_url":"https://api.github.com/users/RAttab/events{/privacy}","received_events_url":"https://api.github.com/users/RAttab/received_events","type":"User","site_admin":false},"repo":{"id":102304552,"name":"rill","full_name":"RAttab/rill","owner":{"login":"RAttab","id":652873,"avatar_url":"https://avatars1.githubusercontent.com/u/652873?v=4","gravatar_id":"","url":"https://api.github.com/users/RAttab","html_url":"https://github.com/RAttab","followers_url":"https://api.github.com/users/RAttab/followers","following_url":"https://api.github.com/users/RAttab/following{/other_user}","gists_url":"https://api.github.com/users/RAttab/gists{/gist_id}","starred_url":"https://api.github.com/users/RAttab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/RAttab/subscriptions","organizations_url":"https://api.github.com/users/RAttab/orgs","repos_url":"https://api.github.com/users/RAttab/repos","events_url":"https://api.github.com/users/RAttab/events{/privacy}","received_events_url":"https://api.github.com/users/RAttab/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/RAttab/rill","description":"one-off database","fork":false,"url":"https://api.github.com/repos/RAttab/rill","forks_url":"https://api.github.com/repos/RAttab/rill/forks","keys_url":"https://api.github.com/repos/RAttab/rill/keys{/key_id}","collaborators_url":"https://api.github.com/repos/RAttab/rill/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/RAttab/rill/teams","hooks_url":"https://api.github.com/repos/RAttab/rill/hooks","issue_events_url":"https://api.github.com/repos/RAttab/rill/issues/events{/number}","events_url":"https://api.github.com/repos/RAttab/rill/events","assignees_url":"https://api.github.com/repos/RAttab/rill/assignees{/user}","branches_url":"https://api.github.com/repos/RAttab/rill/branches{/branch}","tags_url":"https://api.github.com/repos/RAttab/rill/tags","blobs_url":"https://api.github.com/repos/RAttab/rill/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/RAttab/rill/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/RAttab/rill/git/refs{/sha}","trees_url":"https://api.github.com/repos/RAttab/rill/git/trees{/sha}","statuses_url":"https://api.github.com/repos/RAttab/rill/statuses/{sha}","languages_url":"https://api.github.com/repos/RAttab/rill/languages","stargazers_url":"https://api.github.com/repos/RAttab/rill/stargazers","contributors_url":"https://api.github.com/repos/RAttab/rill/contributors","subscribers_url":"https://api.github.com/repos/RAttab/rill/subscribers","subscription_url":"https://api.github.com/repos/RAttab/rill/subscription","commits_url":"https://api.github.com/repos/RAttab/rill/commits{/sha}","git_commits_url":"https://api.github.com/repos/RAttab/rill/git/commits{/sha}","comments_url":"https://api.github.com/repos/RAttab/rill/comments{/number}","issue_comment_url":"https://api.github.com/repos/RAttab/rill/issues/comments{/number}","contents_url":"https://api.github.com/repos/RAttab/rill/contents/{+path}","compare_url":"https://api.github.com/repos/RAttab/rill/compare/{base}...{head}","merges_url":"https://api.github.com/repos/RAttab/rill/merges","archive_url":"https://api.github.com/repos/RAttab/rill/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/RAttab/rill/downloads","issues_url":"https://api.github.com/repos/RAttab/rill/issues{/number}","pulls_url":"https://api.github.com/repos/RAttab/rill/pulls{/number}","milestones_url":"https://api.github.com/repos/RAttab/rill/milestones{/number}","notifications_url":"https://api.github.com/repos/RAttab/rill/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/RAttab/rill/labels{/name}","releases_url":"https://api.github.com/repos/RAttab/rill/releases{/id}","deployments_url":"https://api.github.com/repos/RAttab/rill/deployments","created_at":"2017-09-04T01:00:49Z","updated_at":"2017-12-30T22:01:00Z","pushed_at":"2018-03-24T01:46:57Z","git_url":"git://github.com/RAttab/rill.git","ssh_url":"git@github.com:RAttab/rill.git","clone_url":"https://github.com/RAttab/rill.git","svn_url":"https://github.com/RAttab/rill","homepage":null,"size":92,"stargazers_count":2,"watchers_count":2,"language":"C","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":2,"mirror_url":null,"archived":false,"open_issues_count":3,"license":{"key":"bsd-2-clause","name":"BSD 2-Clause \"Simplified\" License","spdx_id":"BSD-2-Clause","url":"https://api.github.com/licenses/bsd-2-clause"},"forks":2,"open_issues":3,"watchers":2,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/RAttab/rill/pulls/6"},"html":{"href":"https://github.com/RAttab/rill/pull/6"},"issue":{"href":"https://api.github.com/repos/RAttab/rill/issues/6"},"comments":{"href":"https://api.github.com/repos/RAttab/rill/issues/6/comments"},"review_comments":{"href":"https://api.github.com/repos/RAttab/rill/pulls/6/comments"},"review_comment":{"href":"https://api.github.com/repos/RAttab/rill/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/RAttab/rill/pulls/6/commits"},"statuses":{"href":"https://api.github.com/repos/RAttab/rill/statuses/4745928f1613ba6403f525bb503bbae2cf2da27e"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"RAttab"}}