{"action":"created","comment":{"url":"https://api.github.com/repos/BD2KGenomics/toil/pulls/comments/58433230","id":58433230,"diff_hunk":"@@ -174,6 +178,85 @@ def workerCleanup(workerCleanupInfo):\n         if cleanWorkDir == 'always' or ((cleanWorkDir == 'onSuccess' or cleanWorkDir == 'onError') and dirIsEmpty):\n             shutil.rmtree(workflowDir)\n \n+    @staticmethod\n+    def probeUsedDisk(workflowID, workDir=None, **kwargs):\n+        '''\n+        Used to probe the used disk space (on the block device mounting workDir) minus the space\n+        used by Toil. Subtracting this from the total disk size gives the batchsystem an estimate of\n+        how much space is available for Toil. On Mesos, this can be reserved for a non-Toil role.\n+\n+        :param str workflowID: Unique ID for this workflow\n+        :param str workDir: value for --workDir passed by the user.\n+        :param dict **kwargs: Buffer to handle the other objects in the workerCleanupInfo construct\n+        :return: Amount of disk used by non-Toil purposes\n+        '''\n+        workflowDir = getToilWorkflowDir(workflowID, workDir)\n+        toilDiskUsed = AbstractBatchSystem.getWorkflowDirSize(workflowDir)\n+        diskStats = os.statvfs(workflowDir)\n+        # Total Disk used = block size * (total blocks - free blocks)\n+        diskUsed = diskStats.f_frsize * (diskStats.f_blocks - diskStats.f_bavail)\n+        # Disk used for non-toil purposes = Total disk used - Disk used by toil (workDir)\n+        nonToilDiskUsed = diskUsed - toilDiskUsed\n+        return nonToilDiskUsed\n+\n+    @staticmethod\n+    def getWorkflowDirSize(workflowDir):\n+        '''\n+        This method will walk through the workflow directory and return the cumulative filesize in\n+        bytes of all the files in the directory and its subdirectories.\n+\n+        :param workflowDir: path to the workflow working directory\n+        :return: cumulative size in bytes of all files in the directory\n+        '''\n+        totalSize = 0\n+        # The value from running stat on each linked file is equal. To prevent the same file\n+        # from being counted multiple times, we save the inodes of files that have more than one\n+        # nlink associated with them.\n+        linkedFileInodes = set()\n+        for dirPath, dirNames, fileNames in os.walk(workflowDir):\n+            folderSize = AbstractBatchSystem._getDirSize(dirPath, fileNames, linkedFileInodes)","path":"src/toil/batchSystems/abstractBatchSystem.py","position":51,"original_position":51,"commit_id":"64833232385bac9aa29b5df11254b1f7e599ca59","original_commit_id":"64833232385bac9aa29b5df11254b1f7e599ca59","user":{"login":"hannes-ucsc","id":5143256,"avatar_url":"https://avatars.githubusercontent.com/u/5143256?v=3","gravatar_id":"","url":"https://api.github.com/users/hannes-ucsc","html_url":"https://github.com/hannes-ucsc","followers_url":"https://api.github.com/users/hannes-ucsc/followers","following_url":"https://api.github.com/users/hannes-ucsc/following{/other_user}","gists_url":"https://api.github.com/users/hannes-ucsc/gists{/gist_id}","starred_url":"https://api.github.com/users/hannes-ucsc/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hannes-ucsc/subscriptions","organizations_url":"https://api.github.com/users/hannes-ucsc/orgs","repos_url":"https://api.github.com/users/hannes-ucsc/repos","events_url":"https://api.github.com/users/hannes-ucsc/events{/privacy}","received_events_url":"https://api.github.com/users/hannes-ucsc/received_events","type":"User","site_admin":false},"body":"cls. _getDirSize","created_at":"2016-04-04T19:28:13Z","updated_at":"2016-04-04T19:28:13Z","html_url":"https://github.com/BD2KGenomics/toil/pull/707#discussion_r58433230","pull_request_url":"https://api.github.com/repos/BD2KGenomics/toil/pulls/707","_links":{"self":{"href":"https://api.github.com/repos/BD2KGenomics/toil/pulls/comments/58433230"},"html":{"href":"https://github.com/BD2KGenomics/toil/pull/707#discussion_r58433230"},"pull_request":{"href":"https://api.github.com/repos/BD2KGenomics/toil/pulls/707"}}},"pull_request":{"url":"https://api.github.com/repos/BD2KGenomics/toil/pulls/707","id":61786923,"html_url":"https://github.com/BD2KGenomics/toil/pull/707","diff_url":"https://github.com/BD2KGenomics/toil/pull/707.diff","patch_url":"https://github.com/BD2KGenomics/toil/pull/707.patch","issue_url":"https://api.github.com/repos/BD2KGenomics/toil/issues/707","number":707,"state":"open","locked":false,"title":"Implement dynamic mesos disk estimation (resolves #681)","user":{"login":"arkal","id":11653944,"avatar_url":"https://avatars.githubusercontent.com/u/11653944?v=3","gravatar_id":"","url":"https://api.github.com/users/arkal","html_url":"https://github.com/arkal","followers_url":"https://api.github.com/users/arkal/followers","following_url":"https://api.github.com/users/arkal/following{/other_user}","gists_url":"https://api.github.com/users/arkal/gists{/gist_id}","starred_url":"https://api.github.com/users/arkal/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/arkal/subscriptions","organizations_url":"https://api.github.com/users/arkal/orgs","repos_url":"https://api.github.com/users/arkal/repos","events_url":"https://api.github.com/users/arkal/events{/privacy}","received_events_url":"https://api.github.com/users/arkal/received_events","type":"User","site_admin":false},"body":"resolves #681\r\n\r\n1. Executors now probe the used disk after a job and reserve leftovers into an\r\nun-offered role -- `toil-leftover`,\r\n2. The Master address is sent to executor via ToilJob (Needed to send http\r\nrequests to master endpoints),\r\n3. The mesos credentials are also sent to the executor via ToilJob (Needed to authenticate the http requests). Credentials are now an input to toil - by default they look at `/etc/mesos/credentials.txt`.\r\n4. Mesos Tests place the slaves workdir in a location described by TMP, TMPDIR,\r\nor TEMP instead of /tmp (allows for this to be on the mounted volume).\r\n5. Added a test `testDynamicReservation` to batchSystemsTest.py that tests reservation.","created_at":"2016-03-04T23:26:53Z","updated_at":"2016-04-04T19:28:13Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5bdbabfeb92f77a210847f7a75a8f803f47f98e9","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/BD2KGenomics/toil/pulls/707/commits","review_comments_url":"https://api.github.com/repos/BD2KGenomics/toil/pulls/707/comments","review_comment_url":"https://api.github.com/repos/BD2KGenomics/toil/pulls/comments{/number}","comments_url":"https://api.github.com/repos/BD2KGenomics/toil/issues/707/comments","statuses_url":"https://api.github.com/repos/BD2KGenomics/toil/statuses/64833232385bac9aa29b5df11254b1f7e599ca59","head":{"label":"arkal:issues/681-dynamic-mesos-disk","ref":"issues/681-dynamic-mesos-disk","sha":"64833232385bac9aa29b5df11254b1f7e599ca59","user":{"login":"arkal","id":11653944,"avatar_url":"https://avatars.githubusercontent.com/u/11653944?v=3","gravatar_id":"","url":"https://api.github.com/users/arkal","html_url":"https://github.com/arkal","followers_url":"https://api.github.com/users/arkal/followers","following_url":"https://api.github.com/users/arkal/following{/other_user}","gists_url":"https://api.github.com/users/arkal/gists{/gist_id}","starred_url":"https://api.github.com/users/arkal/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/arkal/subscriptions","organizations_url":"https://api.github.com/users/arkal/orgs","repos_url":"https://api.github.com/users/arkal/repos","events_url":"https://api.github.com/users/arkal/events{/privacy}","received_events_url":"https://api.github.com/users/arkal/received_events","type":"User","site_admin":false},"repo":{"id":43851414,"name":"toil","full_name":"arkal/toil","owner":{"login":"arkal","id":11653944,"avatar_url":"https://avatars.githubusercontent.com/u/11653944?v=3","gravatar_id":"","url":"https://api.github.com/users/arkal","html_url":"https://github.com/arkal","followers_url":"https://api.github.com/users/arkal/followers","following_url":"https://api.github.com/users/arkal/following{/other_user}","gists_url":"https://api.github.com/users/arkal/gists{/gist_id}","starred_url":"https://api.github.com/users/arkal/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/arkal/subscriptions","organizations_url":"https://api.github.com/users/arkal/orgs","repos_url":"https://api.github.com/users/arkal/repos","events_url":"https://api.github.com/users/arkal/events{/privacy}","received_events_url":"https://api.github.com/users/arkal/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/arkal/toil","description":"Python based pipeline management software for clusters that makes running recursive and dynamically scheduled computations straightforward. So far works with gridEngine, lsf, parasol and on multi-core machines.","fork":true,"url":"https://api.github.com/repos/arkal/toil","forks_url":"https://api.github.com/repos/arkal/toil/forks","keys_url":"https://api.github.com/repos/arkal/toil/keys{/key_id}","collaborators_url":"https://api.github.com/repos/arkal/toil/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/arkal/toil/teams","hooks_url":"https://api.github.com/repos/arkal/toil/hooks","issue_events_url":"https://api.github.com/repos/arkal/toil/issues/events{/number}","events_url":"https://api.github.com/repos/arkal/toil/events","assignees_url":"https://api.github.com/repos/arkal/toil/assignees{/user}","branches_url":"https://api.github.com/repos/arkal/toil/branches{/branch}","tags_url":"https://api.github.com/repos/arkal/toil/tags","blobs_url":"https://api.github.com/repos/arkal/toil/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/arkal/toil/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/arkal/toil/git/refs{/sha}","trees_url":"https://api.github.com/repos/arkal/toil/git/trees{/sha}","statuses_url":"https://api.github.com/repos/arkal/toil/statuses/{sha}","languages_url":"https://api.github.com/repos/arkal/toil/languages","stargazers_url":"https://api.github.com/repos/arkal/toil/stargazers","contributors_url":"https://api.github.com/repos/arkal/toil/contributors","subscribers_url":"https://api.github.com/repos/arkal/toil/subscribers","subscription_url":"https://api.github.com/repos/arkal/toil/subscription","commits_url":"https://api.github.com/repos/arkal/toil/commits{/sha}","git_commits_url":"https://api.github.com/repos/arkal/toil/git/commits{/sha}","comments_url":"https://api.github.com/repos/arkal/toil/comments{/number}","issue_comment_url":"https://api.github.com/repos/arkal/toil/issues/comments{/number}","contents_url":"https://api.github.com/repos/arkal/toil/contents/{+path}","compare_url":"https://api.github.com/repos/arkal/toil/compare/{base}...{head}","merges_url":"https://api.github.com/repos/arkal/toil/merges","archive_url":"https://api.github.com/repos/arkal/toil/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/arkal/toil/downloads","issues_url":"https://api.github.com/repos/arkal/toil/issues{/number}","pulls_url":"https://api.github.com/repos/arkal/toil/pulls{/number}","milestones_url":"https://api.github.com/repos/arkal/toil/milestones{/number}","notifications_url":"https://api.github.com/repos/arkal/toil/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/arkal/toil/labels{/name}","releases_url":"https://api.github.com/repos/arkal/toil/releases{/id}","deployments_url":"https://api.github.com/repos/arkal/toil/deployments","created_at":"2015-10-07T23:49:59Z","updated_at":"2016-03-02T02:05:38Z","pushed_at":"2016-04-02T01:03:45Z","git_url":"git://github.com/arkal/toil.git","ssh_url":"git@github.com:arkal/toil.git","clone_url":"https://github.com/arkal/toil.git","svn_url":"https://github.com/arkal/toil","homepage":"","size":5139,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"BD2KGenomics:master","ref":"master","sha":"7adce079bbb14382809c4c5ff1e095df60857414","user":{"login":"BD2KGenomics","id":8701293,"avatar_url":"https://avatars.githubusercontent.com/u/8701293?v=3","gravatar_id":"","url":"https://api.github.com/users/BD2KGenomics","html_url":"https://github.com/BD2KGenomics","followers_url":"https://api.github.com/users/BD2KGenomics/followers","following_url":"https://api.github.com/users/BD2KGenomics/following{/other_user}","gists_url":"https://api.github.com/users/BD2KGenomics/gists{/gist_id}","starred_url":"https://api.github.com/users/BD2KGenomics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/BD2KGenomics/subscriptions","organizations_url":"https://api.github.com/users/BD2KGenomics/orgs","repos_url":"https://api.github.com/users/BD2KGenomics/repos","events_url":"https://api.github.com/users/BD2KGenomics/events{/privacy}","received_events_url":"https://api.github.com/users/BD2KGenomics/received_events","type":"Organization","site_admin":false},"repo":{"id":33148320,"name":"toil","full_name":"BD2KGenomics/toil","owner":{"login":"BD2KGenomics","id":8701293,"avatar_url":"https://avatars.githubusercontent.com/u/8701293?v=3","gravatar_id":"","url":"https://api.github.com/users/BD2KGenomics","html_url":"https://github.com/BD2KGenomics","followers_url":"https://api.github.com/users/BD2KGenomics/followers","following_url":"https://api.github.com/users/BD2KGenomics/following{/other_user}","gists_url":"https://api.github.com/users/BD2KGenomics/gists{/gist_id}","starred_url":"https://api.github.com/users/BD2KGenomics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/BD2KGenomics/subscriptions","organizations_url":"https://api.github.com/users/BD2KGenomics/orgs","repos_url":"https://api.github.com/users/BD2KGenomics/repos","events_url":"https://api.github.com/users/BD2KGenomics/events{/privacy}","received_events_url":"https://api.github.com/users/BD2KGenomics/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/BD2KGenomics/toil","description":"Toil is a scalable, efficient, cross-platform pipeline management system, written entirely in Python, and designed around the principles of functional programming.","fork":false,"url":"https://api.github.com/repos/BD2KGenomics/toil","forks_url":"https://api.github.com/repos/BD2KGenomics/toil/forks","keys_url":"https://api.github.com/repos/BD2KGenomics/toil/keys{/key_id}","collaborators_url":"https://api.github.com/repos/BD2KGenomics/toil/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/BD2KGenomics/toil/teams","hooks_url":"https://api.github.com/repos/BD2KGenomics/toil/hooks","issue_events_url":"https://api.github.com/repos/BD2KGenomics/toil/issues/events{/number}","events_url":"https://api.github.com/repos/BD2KGenomics/toil/events","assignees_url":"https://api.github.com/repos/BD2KGenomics/toil/assignees{/user}","branches_url":"https://api.github.com/repos/BD2KGenomics/toil/branches{/branch}","tags_url":"https://api.github.com/repos/BD2KGenomics/toil/tags","blobs_url":"https://api.github.com/repos/BD2KGenomics/toil/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/BD2KGenomics/toil/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/BD2KGenomics/toil/git/refs{/sha}","trees_url":"https://api.github.com/repos/BD2KGenomics/toil/git/trees{/sha}","statuses_url":"https://api.github.com/repos/BD2KGenomics/toil/statuses/{sha}","languages_url":"https://api.github.com/repos/BD2KGenomics/toil/languages","stargazers_url":"https://api.github.com/repos/BD2KGenomics/toil/stargazers","contributors_url":"https://api.github.com/repos/BD2KGenomics/toil/contributors","subscribers_url":"https://api.github.com/repos/BD2KGenomics/toil/subscribers","subscription_url":"https://api.github.com/repos/BD2KGenomics/toil/subscription","commits_url":"https://api.github.com/repos/BD2KGenomics/toil/commits{/sha}","git_commits_url":"https://api.github.com/repos/BD2KGenomics/toil/git/commits{/sha}","comments_url":"https://api.github.com/repos/BD2KGenomics/toil/comments{/number}","issue_comment_url":"https://api.github.com/repos/BD2KGenomics/toil/issues/comments{/number}","contents_url":"https://api.github.com/repos/BD2KGenomics/toil/contents/{+path}","compare_url":"https://api.github.com/repos/BD2KGenomics/toil/compare/{base}...{head}","merges_url":"https://api.github.com/repos/BD2KGenomics/toil/merges","archive_url":"https://api.github.com/repos/BD2KGenomics/toil/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/BD2KGenomics/toil/downloads","issues_url":"https://api.github.com/repos/BD2KGenomics/toil/issues{/number}","pulls_url":"https://api.github.com/repos/BD2KGenomics/toil/pulls{/number}","milestones_url":"https://api.github.com/repos/BD2KGenomics/toil/milestones{/number}","notifications_url":"https://api.github.com/repos/BD2KGenomics/toil/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/BD2KGenomics/toil/labels{/name}","releases_url":"https://api.github.com/repos/BD2KGenomics/toil/releases{/id}","deployments_url":"https://api.github.com/repos/BD2KGenomics/toil/deployments","created_at":"2015-03-30T21:08:58Z","updated_at":"2016-03-26T22:29:06Z","pushed_at":"2016-04-04T18:59:22Z","git_url":"git://github.com/BD2KGenomics/toil.git","ssh_url":"git@github.com:BD2KGenomics/toil.git","clone_url":"https://github.com/BD2KGenomics/toil.git","svn_url":"https://github.com/BD2KGenomics/toil","homepage":"","size":5419,"stargazers_count":32,"watchers_count":32,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":16,"mirror_url":null,"open_issues_count":151,"forks":16,"open_issues":151,"watchers":32,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/BD2KGenomics/toil/pulls/707"},"html":{"href":"https://github.com/BD2KGenomics/toil/pull/707"},"issue":{"href":"https://api.github.com/repos/BD2KGenomics/toil/issues/707"},"comments":{"href":"https://api.github.com/repos/BD2KGenomics/toil/issues/707/comments"},"review_comments":{"href":"https://api.github.com/repos/BD2KGenomics/toil/pulls/707/comments"},"review_comment":{"href":"https://api.github.com/repos/BD2KGenomics/toil/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/BD2KGenomics/toil/pulls/707/commits"},"statuses":{"href":"https://api.github.com/repos/BD2KGenomics/toil/statuses/64833232385bac9aa29b5df11254b1f7e599ca59"}}}}