{"action":"opened","number":513,"pull_request":{"url":"https://api.github.com/repos/HazyResearch/deepdive/pulls/513","id":62101093,"html_url":"https://github.com/HazyResearch/deepdive/pull/513","diff_url":"https://github.com/HazyResearch/deepdive/pull/513.diff","patch_url":"https://github.com/HazyResearch/deepdive/pull/513.patch","issue_url":"https://api.github.com/repos/HazyResearch/deepdive/issues/513","number":513,"state":"open","locked":false,"title":"Support for using Torque scheduler with DeepDive","user":{"login":"igozali","id":1359333,"avatar_url":"https://avatars.githubusercontent.com/u/1359333?v=3","gravatar_id":"","url":"https://api.github.com/users/igozali","html_url":"https://github.com/igozali","followers_url":"https://api.github.com/users/igozali/followers","following_url":"https://api.github.com/users/igozali/following{/other_user}","gists_url":"https://api.github.com/users/igozali/gists{/gist_id}","starred_url":"https://api.github.com/users/igozali/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/igozali/subscriptions","organizations_url":"https://api.github.com/users/igozali/orgs","repos_url":"https://api.github.com/users/igozali/repos","events_url":"https://api.github.com/users/igozali/events{/privacy}","received_events_url":"https://api.github.com/users/igozali/received_events","type":"User","site_admin":false},"body":"This pull request implements a DeepDive extension so that we can execute the processes in the data flow using Torque compiler. Such extensions of this form are called compute-drivers. There are still some cleanup tasks to be performed, but I figured I'd create the PR to get early feedback from all of you! \r\n\r\nPlease see`docs/using-torque.md` directory in this branch for more information on how the interface looks like and how to use it. \r\n\r\nEarly tests show that there are indeed some usefulness of this extension. Using the Infolab compute cluster, I was able to process the whole spouse articles (signalmedia-1m.jsonl, around 200MB after `grep`ing for wife, spouse, etc.) in an hour by using 25 nodes (as defined by PBS). ","created_at":"2016-03-08T16:58:24Z","updated_at":"2016-03-08T16:58:24Z","closed_at":null,"merged_at":null,"merge_commit_sha":null,"assignee":{"login":"igozali","id":1359333,"avatar_url":"https://avatars.githubusercontent.com/u/1359333?v=3","gravatar_id":"","url":"https://api.github.com/users/igozali","html_url":"https://github.com/igozali","followers_url":"https://api.github.com/users/igozali/followers","following_url":"https://api.github.com/users/igozali/following{/other_user}","gists_url":"https://api.github.com/users/igozali/gists{/gist_id}","starred_url":"https://api.github.com/users/igozali/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/igozali/subscriptions","organizations_url":"https://api.github.com/users/igozali/orgs","repos_url":"https://api.github.com/users/igozali/repos","events_url":"https://api.github.com/users/igozali/events{/privacy}","received_events_url":"https://api.github.com/users/igozali/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/HazyResearch/deepdive/pulls/513/commits","review_comments_url":"https://api.github.com/repos/HazyResearch/deepdive/pulls/513/comments","review_comment_url":"https://api.github.com/repos/HazyResearch/deepdive/pulls/comments{/number}","comments_url":"https://api.github.com/repos/HazyResearch/deepdive/issues/513/comments","statuses_url":"https://api.github.com/repos/HazyResearch/deepdive/statuses/54cf31664f6ef9538e9eb67d5dd9b087cd7e1db8","head":{"label":"HazyResearch:igozali-torque-computer","ref":"igozali-torque-computer","sha":"54cf31664f6ef9538e9eb67d5dd9b087cd7e1db8","user":{"login":"HazyResearch","id":2165246,"avatar_url":"https://avatars.githubusercontent.com/u/2165246?v=3","gravatar_id":"","url":"https://api.github.com/users/HazyResearch","html_url":"https://github.com/HazyResearch","followers_url":"https://api.github.com/users/HazyResearch/followers","following_url":"https://api.github.com/users/HazyResearch/following{/other_user}","gists_url":"https://api.github.com/users/HazyResearch/gists{/gist_id}","starred_url":"https://api.github.com/users/HazyResearch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/HazyResearch/subscriptions","organizations_url":"https://api.github.com/users/HazyResearch/orgs","repos_url":"https://api.github.com/users/HazyResearch/repos","events_url":"https://api.github.com/users/HazyResearch/events{/privacy}","received_events_url":"https://api.github.com/users/HazyResearch/received_events","type":"Organization","site_admin":false},"repo":{"id":17907002,"name":"deepdive","full_name":"HazyResearch/deepdive","owner":{"login":"HazyResearch","id":2165246,"avatar_url":"https://avatars.githubusercontent.com/u/2165246?v=3","gravatar_id":"","url":"https://api.github.com/users/HazyResearch","html_url":"https://github.com/HazyResearch","followers_url":"https://api.github.com/users/HazyResearch/followers","following_url":"https://api.github.com/users/HazyResearch/following{/other_user}","gists_url":"https://api.github.com/users/HazyResearch/gists{/gist_id}","starred_url":"https://api.github.com/users/HazyResearch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/HazyResearch/subscriptions","organizations_url":"https://api.github.com/users/HazyResearch/orgs","repos_url":"https://api.github.com/users/HazyResearch/repos","events_url":"https://api.github.com/users/HazyResearch/events{/privacy}","received_events_url":"https://api.github.com/users/HazyResearch/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/HazyResearch/deepdive","description":"DeepDive","fork":false,"url":"https://api.github.com/repos/HazyResearch/deepdive","forks_url":"https://api.github.com/repos/HazyResearch/deepdive/forks","keys_url":"https://api.github.com/repos/HazyResearch/deepdive/keys{/key_id}","collaborators_url":"https://api.github.com/repos/HazyResearch/deepdive/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/HazyResearch/deepdive/teams","hooks_url":"https://api.github.com/repos/HazyResearch/deepdive/hooks","issue_events_url":"https://api.github.com/repos/HazyResearch/deepdive/issues/events{/number}","events_url":"https://api.github.com/repos/HazyResearch/deepdive/events","assignees_url":"https://api.github.com/repos/HazyResearch/deepdive/assignees{/user}","branches_url":"https://api.github.com/repos/HazyResearch/deepdive/branches{/branch}","tags_url":"https://api.github.com/repos/HazyResearch/deepdive/tags","blobs_url":"https://api.github.com/repos/HazyResearch/deepdive/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/HazyResearch/deepdive/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/HazyResearch/deepdive/git/refs{/sha}","trees_url":"https://api.github.com/repos/HazyResearch/deepdive/git/trees{/sha}","statuses_url":"https://api.github.com/repos/HazyResearch/deepdive/statuses/{sha}","languages_url":"https://api.github.com/repos/HazyResearch/deepdive/languages","stargazers_url":"https://api.github.com/repos/HazyResearch/deepdive/stargazers","contributors_url":"https://api.github.com/repos/HazyResearch/deepdive/contributors","subscribers_url":"https://api.github.com/repos/HazyResearch/deepdive/subscribers","subscription_url":"https://api.github.com/repos/HazyResearch/deepdive/subscription","commits_url":"https://api.github.com/repos/HazyResearch/deepdive/commits{/sha}","git_commits_url":"https://api.github.com/repos/HazyResearch/deepdive/git/commits{/sha}","comments_url":"https://api.github.com/repos/HazyResearch/deepdive/comments{/number}","issue_comment_url":"https://api.github.com/repos/HazyResearch/deepdive/issues/comments{/number}","contents_url":"https://api.github.com/repos/HazyResearch/deepdive/contents/{+path}","compare_url":"https://api.github.com/repos/HazyResearch/deepdive/compare/{base}...{head}","merges_url":"https://api.github.com/repos/HazyResearch/deepdive/merges","archive_url":"https://api.github.com/repos/HazyResearch/deepdive/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/HazyResearch/deepdive/downloads","issues_url":"https://api.github.com/repos/HazyResearch/deepdive/issues{/number}","pulls_url":"https://api.github.com/repos/HazyResearch/deepdive/pulls{/number}","milestones_url":"https://api.github.com/repos/HazyResearch/deepdive/milestones{/number}","notifications_url":"https://api.github.com/repos/HazyResearch/deepdive/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/HazyResearch/deepdive/labels{/name}","releases_url":"https://api.github.com/repos/HazyResearch/deepdive/releases{/id}","deployments_url":"https://api.github.com/repos/HazyResearch/deepdive/deployments","created_at":"2014-03-19T14:23:50Z","updated_at":"2016-03-07T22:33:38Z","pushed_at":"2016-03-08T16:58:24Z","git_url":"git://github.com/HazyResearch/deepdive.git","ssh_url":"git@github.com:HazyResearch/deepdive.git","clone_url":"https://github.com/HazyResearch/deepdive.git","svn_url":"https://github.com/HazyResearch/deepdive","homepage":"deepdive.stanford.edu","size":488351,"stargazers_count":812,"watchers_count":812,"language":"Shell","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":225,"mirror_url":null,"open_issues_count":46,"forks":225,"open_issues":46,"watchers":812,"default_branch":"master"}},"base":{"label":"HazyResearch:master","ref":"master","sha":"c8cf30f4c550d48244b8a0956eabdc9058b35a62","user":{"login":"HazyResearch","id":2165246,"avatar_url":"https://avatars.githubusercontent.com/u/2165246?v=3","gravatar_id":"","url":"https://api.github.com/users/HazyResearch","html_url":"https://github.com/HazyResearch","followers_url":"https://api.github.com/users/HazyResearch/followers","following_url":"https://api.github.com/users/HazyResearch/following{/other_user}","gists_url":"https://api.github.com/users/HazyResearch/gists{/gist_id}","starred_url":"https://api.github.com/users/HazyResearch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/HazyResearch/subscriptions","organizations_url":"https://api.github.com/users/HazyResearch/orgs","repos_url":"https://api.github.com/users/HazyResearch/repos","events_url":"https://api.github.com/users/HazyResearch/events{/privacy}","received_events_url":"https://api.github.com/users/HazyResearch/received_events","type":"Organization","site_admin":false},"repo":{"id":17907002,"name":"deepdive","full_name":"HazyResearch/deepdive","owner":{"login":"HazyResearch","id":2165246,"avatar_url":"https://avatars.githubusercontent.com/u/2165246?v=3","gravatar_id":"","url":"https://api.github.com/users/HazyResearch","html_url":"https://github.com/HazyResearch","followers_url":"https://api.github.com/users/HazyResearch/followers","following_url":"https://api.github.com/users/HazyResearch/following{/other_user}","gists_url":"https://api.github.com/users/HazyResearch/gists{/gist_id}","starred_url":"https://api.github.com/users/HazyResearch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/HazyResearch/subscriptions","organizations_url":"https://api.github.com/users/HazyResearch/orgs","repos_url":"https://api.github.com/users/HazyResearch/repos","events_url":"https://api.github.com/users/HazyResearch/events{/privacy}","received_events_url":"https://api.github.com/users/HazyResearch/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/HazyResearch/deepdive","description":"DeepDive","fork":false,"url":"https://api.github.com/repos/HazyResearch/deepdive","forks_url":"https://api.github.com/repos/HazyResearch/deepdive/forks","keys_url":"https://api.github.com/repos/HazyResearch/deepdive/keys{/key_id}","collaborators_url":"https://api.github.com/repos/HazyResearch/deepdive/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/HazyResearch/deepdive/teams","hooks_url":"https://api.github.com/repos/HazyResearch/deepdive/hooks","issue_events_url":"https://api.github.com/repos/HazyResearch/deepdive/issues/events{/number}","events_url":"https://api.github.com/repos/HazyResearch/deepdive/events","assignees_url":"https://api.github.com/repos/HazyResearch/deepdive/assignees{/user}","branches_url":"https://api.github.com/repos/HazyResearch/deepdive/branches{/branch}","tags_url":"https://api.github.com/repos/HazyResearch/deepdive/tags","blobs_url":"https://api.github.com/repos/HazyResearch/deepdive/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/HazyResearch/deepdive/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/HazyResearch/deepdive/git/refs{/sha}","trees_url":"https://api.github.com/repos/HazyResearch/deepdive/git/trees{/sha}","statuses_url":"https://api.github.com/repos/HazyResearch/deepdive/statuses/{sha}","languages_url":"https://api.github.com/repos/HazyResearch/deepdive/languages","stargazers_url":"https://api.github.com/repos/HazyResearch/deepdive/stargazers","contributors_url":"https://api.github.com/repos/HazyResearch/deepdive/contributors","subscribers_url":"https://api.github.com/repos/HazyResearch/deepdive/subscribers","subscription_url":"https://api.github.com/repos/HazyResearch/deepdive/subscription","commits_url":"https://api.github.com/repos/HazyResearch/deepdive/commits{/sha}","git_commits_url":"https://api.github.com/repos/HazyResearch/deepdive/git/commits{/sha}","comments_url":"https://api.github.com/repos/HazyResearch/deepdive/comments{/number}","issue_comment_url":"https://api.github.com/repos/HazyResearch/deepdive/issues/comments{/number}","contents_url":"https://api.github.com/repos/HazyResearch/deepdive/contents/{+path}","compare_url":"https://api.github.com/repos/HazyResearch/deepdive/compare/{base}...{head}","merges_url":"https://api.github.com/repos/HazyResearch/deepdive/merges","archive_url":"https://api.github.com/repos/HazyResearch/deepdive/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/HazyResearch/deepdive/downloads","issues_url":"https://api.github.com/repos/HazyResearch/deepdive/issues{/number}","pulls_url":"https://api.github.com/repos/HazyResearch/deepdive/pulls{/number}","milestones_url":"https://api.github.com/repos/HazyResearch/deepdive/milestones{/number}","notifications_url":"https://api.github.com/repos/HazyResearch/deepdive/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/HazyResearch/deepdive/labels{/name}","releases_url":"https://api.github.com/repos/HazyResearch/deepdive/releases{/id}","deployments_url":"https://api.github.com/repos/HazyResearch/deepdive/deployments","created_at":"2014-03-19T14:23:50Z","updated_at":"2016-03-07T22:33:38Z","pushed_at":"2016-03-08T16:58:24Z","git_url":"git://github.com/HazyResearch/deepdive.git","ssh_url":"git@github.com:HazyResearch/deepdive.git","clone_url":"https://github.com/HazyResearch/deepdive.git","svn_url":"https://github.com/HazyResearch/deepdive","homepage":"deepdive.stanford.edu","size":488351,"stargazers_count":812,"watchers_count":812,"language":"Shell","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":225,"mirror_url":null,"open_issues_count":46,"forks":225,"open_issues":46,"watchers":812,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/HazyResearch/deepdive/pulls/513"},"html":{"href":"https://github.com/HazyResearch/deepdive/pull/513"},"issue":{"href":"https://api.github.com/repos/HazyResearch/deepdive/issues/513"},"comments":{"href":"https://api.github.com/repos/HazyResearch/deepdive/issues/513/comments"},"review_comments":{"href":"https://api.github.com/repos/HazyResearch/deepdive/pulls/513/comments"},"review_comment":{"href":"https://api.github.com/repos/HazyResearch/deepdive/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/HazyResearch/deepdive/pulls/513/commits"},"statuses":{"href":"https://api.github.com/repos/HazyResearch/deepdive/statuses/54cf31664f6ef9538e9eb67d5dd9b087cd7e1db8"}},"merged":false,"mergeable":null,"mergeable_state":"unknown","merged_by":null,"comments":0,"review_comments":0,"commits":24,"additions":605,"deletions":1,"changed_files":13}}