{"action":"created","comment":{"url":"https://api.github.com/repos/mldbai/mldb/pulls/comments/57917706","id":57917706,"diff_hunk":"@@ -20,6 +31,65 @@ namespace Datacratic {\n namespace MLDB {\n \n static constexpr size_t TABULAR_DATASET_DEFAULT_ROWS_PER_CHUNK=65536;\n+static constexpr size_t NUM_PARALLEL_CHUNKS=16;\n+\n+namespace {\n+\n+// Note: in GCC 4.9+, we can use the std::atomic_xxx overloads for\n+// std::shared_ptr.  Once the Concurrency TR is available, we can\n+// replace with those classes.  For the moment we use a simple,\n+// spinlock protected implementation that is a lowest common\n+// denominator.\n+template<typename T>\n+struct atomic_shared_ptr {","path":"plugins/tabular_dataset.cc","position":37,"original_position":37,"commit_id":"f4883d70d4f22f2300c18ec9b2f1bf8032d5c6d1","original_commit_id":"f4883d70d4f22f2300c18ec9b2f1bf8032d5c6d1","user":{"login":"guyd","id":683788,"avatar_url":"https://avatars.githubusercontent.com/u/683788?v=3","gravatar_id":"","url":"https://api.github.com/users/guyd","html_url":"https://github.com/guyd","followers_url":"https://api.github.com/users/guyd/followers","following_url":"https://api.github.com/users/guyd/following{/other_user}","gists_url":"https://api.github.com/users/guyd/gists{/gist_id}","starred_url":"https://api.github.com/users/guyd/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/guyd/subscriptions","organizations_url":"https://api.github.com/users/guyd/orgs","repos_url":"https://api.github.com/users/guyd/repos","events_url":"https://api.github.com/users/guyd/events{/privacy}","received_events_url":"https://api.github.com/users/guyd/received_events","type":"User","site_admin":false},"body":"Consider moving this code to a separate header.","created_at":"2016-03-30T16:17:38Z","updated_at":"2016-03-30T16:17:38Z","html_url":"https://github.com/mldbai/mldb/pull/287#discussion_r57917706","pull_request_url":"https://api.github.com/repos/mldbai/mldb/pulls/287","_links":{"self":{"href":"https://api.github.com/repos/mldbai/mldb/pulls/comments/57917706"},"html":{"href":"https://github.com/mldbai/mldb/pull/287#discussion_r57917706"},"pull_request":{"href":"https://api.github.com/repos/mldbai/mldb/pulls/287"}}},"pull_request":{"url":"https://api.github.com/repos/mldbai/mldb/pulls/287","id":64628952,"html_url":"https://github.com/mldbai/mldb/pull/287","diff_url":"https://github.com/mldbai/mldb/pull/287.diff","patch_url":"https://github.com/mldbai/mldb/pull/287.patch","issue_url":"https://api.github.com/repos/mldbai/mldb/issues/287","number":287,"state":"open","locked":false,"title":"MLDB-1507 bid request import","user":{"login":"jeremybarnes","id":112556,"avatar_url":"https://avatars.githubusercontent.com/u/112556?v=3","gravatar_id":"","url":"https://api.github.com/users/jeremybarnes","html_url":"https://github.com/jeremybarnes","followers_url":"https://api.github.com/users/jeremybarnes/followers","following_url":"https://api.github.com/users/jeremybarnes/following{/other_user}","gists_url":"https://api.github.com/users/jeremybarnes/gists{/gist_id}","starred_url":"https://api.github.com/users/jeremybarnes/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jeremybarnes/subscriptions","organizations_url":"https://api.github.com/users/jeremybarnes/orgs","repos_url":"https://api.github.com/users/jeremybarnes/repos","events_url":"https://api.github.com/users/jeremybarnes/events{/privacy}","received_events_url":"https://api.github.com/users/jeremybarnes/received_events","type":"User","site_admin":false},"body":"This improves the importation of JSON-structured data like bid requests.  It has the following two main effects:\r\n\r\n1.  Bid request import throughput goes from ~15,000 per second to ~75,000 per second\r\n2.  Memory usage of bid requests is roughly 2x the gzipped file size, rather than roughly 50x the gzipped file size\r\n\r\nIt works by adding the ability for the Tabular dataset to have an open schema, and adding a Recorder interface which can be requested by something that's bulk inserting and can be specialized when the schema (or part of the schema) is known.\r\n","created_at":"2016-03-30T13:42:12Z","updated_at":"2016-03-30T16:17:38Z","closed_at":null,"merged_at":null,"merge_commit_sha":"a62b02131112b7449c66c4cc753a161c4142b933","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/mldbai/mldb/pulls/287/commits","review_comments_url":"https://api.github.com/repos/mldbai/mldb/pulls/287/comments","review_comment_url":"https://api.github.com/repos/mldbai/mldb/pulls/comments{/number}","comments_url":"https://api.github.com/repos/mldbai/mldb/issues/287/comments","statuses_url":"https://api.github.com/repos/mldbai/mldb/statuses/f4883d70d4f22f2300c18ec9b2f1bf8032d5c6d1","head":{"label":"mldbai:MLDB-1507-bid-request-import","ref":"MLDB-1507-bid-request-import","sha":"f4883d70d4f22f2300c18ec9b2f1bf8032d5c6d1","user":{"login":"mldbai","id":11510172,"avatar_url":"https://avatars.githubusercontent.com/u/11510172?v=3","gravatar_id":"","url":"https://api.github.com/users/mldbai","html_url":"https://github.com/mldbai","followers_url":"https://api.github.com/users/mldbai/followers","following_url":"https://api.github.com/users/mldbai/following{/other_user}","gists_url":"https://api.github.com/users/mldbai/gists{/gist_id}","starred_url":"https://api.github.com/users/mldbai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mldbai/subscriptions","organizations_url":"https://api.github.com/users/mldbai/orgs","repos_url":"https://api.github.com/users/mldbai/repos","events_url":"https://api.github.com/users/mldbai/events{/privacy}","received_events_url":"https://api.github.com/users/mldbai/received_events","type":"Organization","site_admin":false},"repo":{"id":47634692,"name":"mldb","full_name":"mldbai/mldb","owner":{"login":"mldbai","id":11510172,"avatar_url":"https://avatars.githubusercontent.com/u/11510172?v=3","gravatar_id":"","url":"https://api.github.com/users/mldbai","html_url":"https://github.com/mldbai","followers_url":"https://api.github.com/users/mldbai/followers","following_url":"https://api.github.com/users/mldbai/following{/other_user}","gists_url":"https://api.github.com/users/mldbai/gists{/gist_id}","starred_url":"https://api.github.com/users/mldbai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mldbai/subscriptions","organizations_url":"https://api.github.com/users/mldbai/orgs","repos_url":"https://api.github.com/users/mldbai/repos","events_url":"https://api.github.com/users/mldbai/events{/privacy}","received_events_url":"https://api.github.com/users/mldbai/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mldbai/mldb","description":"MLDB is the Machine Learning Database","fork":false,"url":"https://api.github.com/repos/mldbai/mldb","forks_url":"https://api.github.com/repos/mldbai/mldb/forks","keys_url":"https://api.github.com/repos/mldbai/mldb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mldbai/mldb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mldbai/mldb/teams","hooks_url":"https://api.github.com/repos/mldbai/mldb/hooks","issue_events_url":"https://api.github.com/repos/mldbai/mldb/issues/events{/number}","events_url":"https://api.github.com/repos/mldbai/mldb/events","assignees_url":"https://api.github.com/repos/mldbai/mldb/assignees{/user}","branches_url":"https://api.github.com/repos/mldbai/mldb/branches{/branch}","tags_url":"https://api.github.com/repos/mldbai/mldb/tags","blobs_url":"https://api.github.com/repos/mldbai/mldb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mldbai/mldb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mldbai/mldb/git/refs{/sha}","trees_url":"https://api.github.com/repos/mldbai/mldb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mldbai/mldb/statuses/{sha}","languages_url":"https://api.github.com/repos/mldbai/mldb/languages","stargazers_url":"https://api.github.com/repos/mldbai/mldb/stargazers","contributors_url":"https://api.github.com/repos/mldbai/mldb/contributors","subscribers_url":"https://api.github.com/repos/mldbai/mldb/subscribers","subscription_url":"https://api.github.com/repos/mldbai/mldb/subscription","commits_url":"https://api.github.com/repos/mldbai/mldb/commits{/sha}","git_commits_url":"https://api.github.com/repos/mldbai/mldb/git/commits{/sha}","comments_url":"https://api.github.com/repos/mldbai/mldb/comments{/number}","issue_comment_url":"https://api.github.com/repos/mldbai/mldb/issues/comments{/number}","contents_url":"https://api.github.com/repos/mldbai/mldb/contents/{+path}","compare_url":"https://api.github.com/repos/mldbai/mldb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mldbai/mldb/merges","archive_url":"https://api.github.com/repos/mldbai/mldb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mldbai/mldb/downloads","issues_url":"https://api.github.com/repos/mldbai/mldb/issues{/number}","pulls_url":"https://api.github.com/repos/mldbai/mldb/pulls{/number}","milestones_url":"https://api.github.com/repos/mldbai/mldb/milestones{/number}","notifications_url":"https://api.github.com/repos/mldbai/mldb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mldbai/mldb/labels{/name}","releases_url":"https://api.github.com/repos/mldbai/mldb/releases{/id}","deployments_url":"https://api.github.com/repos/mldbai/mldb/deployments","created_at":"2015-12-08T16:34:16Z","updated_at":"2016-03-21T11:05:10Z","pushed_at":"2016-03-30T14:55:03Z","git_url":"git://github.com/mldbai/mldb.git","ssh_url":"git@github.com:mldbai/mldb.git","clone_url":"https://github.com/mldbai/mldb.git","svn_url":"https://github.com/mldbai/mldb","homepage":"http://mldb.ai","size":15228,"stargazers_count":27,"watchers_count":27,"language":"C++","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":3,"mirror_url":null,"open_issues_count":9,"forks":3,"open_issues":9,"watchers":27,"default_branch":"master"}},"base":{"label":"mldbai:master","ref":"master","sha":"d339ad1bd25e1bd1f2e2cf62786b1ee1930d7ce3","user":{"login":"mldbai","id":11510172,"avatar_url":"https://avatars.githubusercontent.com/u/11510172?v=3","gravatar_id":"","url":"https://api.github.com/users/mldbai","html_url":"https://github.com/mldbai","followers_url":"https://api.github.com/users/mldbai/followers","following_url":"https://api.github.com/users/mldbai/following{/other_user}","gists_url":"https://api.github.com/users/mldbai/gists{/gist_id}","starred_url":"https://api.github.com/users/mldbai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mldbai/subscriptions","organizations_url":"https://api.github.com/users/mldbai/orgs","repos_url":"https://api.github.com/users/mldbai/repos","events_url":"https://api.github.com/users/mldbai/events{/privacy}","received_events_url":"https://api.github.com/users/mldbai/received_events","type":"Organization","site_admin":false},"repo":{"id":47634692,"name":"mldb","full_name":"mldbai/mldb","owner":{"login":"mldbai","id":11510172,"avatar_url":"https://avatars.githubusercontent.com/u/11510172?v=3","gravatar_id":"","url":"https://api.github.com/users/mldbai","html_url":"https://github.com/mldbai","followers_url":"https://api.github.com/users/mldbai/followers","following_url":"https://api.github.com/users/mldbai/following{/other_user}","gists_url":"https://api.github.com/users/mldbai/gists{/gist_id}","starred_url":"https://api.github.com/users/mldbai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mldbai/subscriptions","organizations_url":"https://api.github.com/users/mldbai/orgs","repos_url":"https://api.github.com/users/mldbai/repos","events_url":"https://api.github.com/users/mldbai/events{/privacy}","received_events_url":"https://api.github.com/users/mldbai/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mldbai/mldb","description":"MLDB is the Machine Learning Database","fork":false,"url":"https://api.github.com/repos/mldbai/mldb","forks_url":"https://api.github.com/repos/mldbai/mldb/forks","keys_url":"https://api.github.com/repos/mldbai/mldb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mldbai/mldb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mldbai/mldb/teams","hooks_url":"https://api.github.com/repos/mldbai/mldb/hooks","issue_events_url":"https://api.github.com/repos/mldbai/mldb/issues/events{/number}","events_url":"https://api.github.com/repos/mldbai/mldb/events","assignees_url":"https://api.github.com/repos/mldbai/mldb/assignees{/user}","branches_url":"https://api.github.com/repos/mldbai/mldb/branches{/branch}","tags_url":"https://api.github.com/repos/mldbai/mldb/tags","blobs_url":"https://api.github.com/repos/mldbai/mldb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mldbai/mldb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mldbai/mldb/git/refs{/sha}","trees_url":"https://api.github.com/repos/mldbai/mldb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mldbai/mldb/statuses/{sha}","languages_url":"https://api.github.com/repos/mldbai/mldb/languages","stargazers_url":"https://api.github.com/repos/mldbai/mldb/stargazers","contributors_url":"https://api.github.com/repos/mldbai/mldb/contributors","subscribers_url":"https://api.github.com/repos/mldbai/mldb/subscribers","subscription_url":"https://api.github.com/repos/mldbai/mldb/subscription","commits_url":"https://api.github.com/repos/mldbai/mldb/commits{/sha}","git_commits_url":"https://api.github.com/repos/mldbai/mldb/git/commits{/sha}","comments_url":"https://api.github.com/repos/mldbai/mldb/comments{/number}","issue_comment_url":"https://api.github.com/repos/mldbai/mldb/issues/comments{/number}","contents_url":"https://api.github.com/repos/mldbai/mldb/contents/{+path}","compare_url":"https://api.github.com/repos/mldbai/mldb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mldbai/mldb/merges","archive_url":"https://api.github.com/repos/mldbai/mldb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mldbai/mldb/downloads","issues_url":"https://api.github.com/repos/mldbai/mldb/issues{/number}","pulls_url":"https://api.github.com/repos/mldbai/mldb/pulls{/number}","milestones_url":"https://api.github.com/repos/mldbai/mldb/milestones{/number}","notifications_url":"https://api.github.com/repos/mldbai/mldb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mldbai/mldb/labels{/name}","releases_url":"https://api.github.com/repos/mldbai/mldb/releases{/id}","deployments_url":"https://api.github.com/repos/mldbai/mldb/deployments","created_at":"2015-12-08T16:34:16Z","updated_at":"2016-03-21T11:05:10Z","pushed_at":"2016-03-30T14:55:03Z","git_url":"git://github.com/mldbai/mldb.git","ssh_url":"git@github.com:mldbai/mldb.git","clone_url":"https://github.com/mldbai/mldb.git","svn_url":"https://github.com/mldbai/mldb","homepage":"http://mldb.ai","size":15228,"stargazers_count":27,"watchers_count":27,"language":"C++","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":3,"mirror_url":null,"open_issues_count":9,"forks":3,"open_issues":9,"watchers":27,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/mldbai/mldb/pulls/287"},"html":{"href":"https://github.com/mldbai/mldb/pull/287"},"issue":{"href":"https://api.github.com/repos/mldbai/mldb/issues/287"},"comments":{"href":"https://api.github.com/repos/mldbai/mldb/issues/287/comments"},"review_comments":{"href":"https://api.github.com/repos/mldbai/mldb/pulls/287/comments"},"review_comment":{"href":"https://api.github.com/repos/mldbai/mldb/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/mldbai/mldb/pulls/287/commits"},"statuses":{"href":"https://api.github.com/repos/mldbai/mldb/statuses/f4883d70d4f22f2300c18ec9b2f1bf8032d5c6d1"}}}}