{"action":"created","comment":{"url":"https://api.github.com/repos/BD2KGenomics/toil/pulls/comments/61381862","id":61381862,"diff_hunk":"@@ -16,62 +16,83 @@\n \n import uuid\n \n-import shutil\n-\n-import os\n from toil.common import Toil\n from toil.job import Job\n-from toil.test import ToilTest\n-import tempfile\n+from toil.leader import FailedJobsException\n+from toil.test import ToilTest, make_tests\n \n \n class ImportExportFileTest(ToilTest):\n     def setUp(self):\n-        self.srcFile = \"%s/%s\" % (tempfile.mkdtemp(), uuid.uuid4())\n-        with open(self.srcFile, 'w') as f:\n-            f.write(os.urandom(2**10))\n-\n-    def tearDown(self):\n-        shutil.rmtree(self.srcFile.rsplit('/', 1)[0])\n+        super(ImportExportFileTest, self).setUp()\n+        self._tempDir = self._createTempDir()\n+        self.dstFile = '%s/%s' % (self._tempDir, 'out')\n \n-    def testImportExportFile(self):\n+    def _importExportFile(self, options, fail):\n+        with Toil(options) as toil:\n+            if not options.restart:\n+                # stage files\n+                srcFile = '%s/%s%s' % (self._tempDir, 'in', uuid.uuid4())\n+                with open(srcFile, 'w') as f:\n+                    f.write('Hello')\n+                inputFileID = toil.importFile('file://' + srcFile)\n+\n+                # write fail value and sets self.failFileID to its ID\n+                # this ID will be passed to the job. The boolean value\n+                # written to it determines if the job fails or not.\n+                with toil._jobStore.writeFileStream() as (failToggle, failFileID):\n+                    self.failFileID = failFileID\n+                    failToggle.write(str(fail))\n+\n+                # run workflow\n+                outputFileID = toil.start(HelloWorld(inputFileID, self.failFileID))\n+            else:\n+                # write fail value\n+                # see above comment for explanation\n+                with toil._jobStore.updateFileStream(self.failFileID) as failToggle:\n+                    failToggle.write('False')\n+\n+                # run workflow\n+                outputFileID = toil.restart()\n+\n+            # export file to dstFile\n+            toil.exportFile(outputFileID, 'file://' + self.dstFile)\n+            with open(self.dstFile, 'r') as f:\n+                assert f.read() == \"HelloWorld!\"\n+\n+\n+    def _importExport(self, restart):\n+        # create options for workflow\n         options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())\n         options.logLevel = \"INFO\"\n-        job = HelloWorld()\n-        with Toil(options) as toil:\n-            jobStoreFileID = toil.importFile('file://%s' % self.srcFile)\n-            assert toil.jobStore.fileExists(jobStoreFileID)\n-\n-            destPath = '%s/%s' % (self.srcFile.rsplit('/', 1)[0], uuid.uuid4())\n-            toil.exportFile(jobStoreFileID, 'file://%s' % destPath)\n-            assert os.path.exists(destPath)\n \n-            toil.run(job)\n+        if restart:\n+            try:\n+                self._importExportFile(options, fail=True)\n+            except FailedJobsException:\n+                options.restart = True\n \n+        self._importExportFile(options, fail=False)\n \n-class HelloWorld(Job):\n-    def __init__(self):\n-        Job.__init__(self,  memory=100000, cores=2, disk=\"3G\")\n-\n-    def run(self, fileStore):\n-        fileID = self.addChildJobFn(childFn, cores=1, memory=\"1M\", disk=\"1M\").rv()\n-        self.addFollowOn(FollowOn(fileID))\n+    @classmethod\n+    def make(cls):","path":"src/toil/test/src/importExportFileTest.py","position":91,"original_position":91,"commit_id":"e2f2dda939286a89e4bf850c82b8a8762ae95dcc","original_commit_id":"e2f2dda939286a89e4bf850c82b8a8762ae95dcc","user":{"login":"hannes-ucsc","id":5143256,"avatar_url":"https://avatars.githubusercontent.com/u/5143256?v=3","gravatar_id":"","url":"https://api.github.com/users/hannes-ucsc","html_url":"https://github.com/hannes-ucsc","followers_url":"https://api.github.com/users/hannes-ucsc/followers","following_url":"https://api.github.com/users/hannes-ucsc/following{/other_user}","gists_url":"https://api.github.com/users/hannes-ucsc/gists{/gist_id}","starred_url":"https://api.github.com/users/hannes-ucsc/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hannes-ucsc/subscriptions","organizations_url":"https://api.github.com/users/hannes-ucsc/orgs","repos_url":"https://api.github.com/users/hannes-ucsc/repos","events_url":"https://api.github.com/users/hannes-ucsc/events{/privacy}","received_events_url":"https://api.github.com/users/hannes-ucsc/received_events","type":"User","site_admin":false},"body":"I think with just two cases, it will be hard to motivate the use of generative programming. Just write these two test methods by hand so you don't have to explain too much when you demo this on Friday.","created_at":"2016-04-28T07:10:07Z","updated_at":"2016-04-28T07:10:07Z","html_url":"https://github.com/BD2KGenomics/toil/pull/769#discussion_r61381862","pull_request_url":"https://api.github.com/repos/BD2KGenomics/toil/pulls/769","_links":{"self":{"href":"https://api.github.com/repos/BD2KGenomics/toil/pulls/comments/61381862"},"html":{"href":"https://github.com/BD2KGenomics/toil/pull/769#discussion_r61381862"},"pull_request":{"href":"https://api.github.com/repos/BD2KGenomics/toil/pulls/769"}}},"pull_request":{"url":"https://api.github.com/repos/BD2KGenomics/toil/pulls/769","id":65748483,"html_url":"https://github.com/BD2KGenomics/toil/pull/769","diff_url":"https://github.com/BD2KGenomics/toil/pull/769.diff","patch_url":"https://github.com/BD2KGenomics/toil/pull/769.patch","issue_url":"https://api.github.com/repos/BD2KGenomics/toil/issues/769","number":769,"state":"open","locked":false,"title":"Improved file staging docs and tests (revolves #766)","user":{"login":"JakeNarkizian","id":10471159,"avatar_url":"https://avatars.githubusercontent.com/u/10471159?v=3","gravatar_id":"","url":"https://api.github.com/users/JakeNarkizian","html_url":"https://github.com/JakeNarkizian","followers_url":"https://api.github.com/users/JakeNarkizian/followers","following_url":"https://api.github.com/users/JakeNarkizian/following{/other_user}","gists_url":"https://api.github.com/users/JakeNarkizian/gists{/gist_id}","starred_url":"https://api.github.com/users/JakeNarkizian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/JakeNarkizian/subscriptions","organizations_url":"https://api.github.com/users/JakeNarkizian/orgs","repos_url":"https://api.github.com/users/JakeNarkizian/repos","events_url":"https://api.github.com/users/JakeNarkizian/events{/privacy}","received_events_url":"https://api.github.com/users/JakeNarkizian/received_events","type":"User","site_admin":false},"body":"Resolves #766\r\n\r\nThe importExportFileTest now spawns a job that creates an output file. This output file is then exported.","created_at":"2016-04-08T07:48:47Z","updated_at":"2016-04-28T07:10:07Z","closed_at":null,"merged_at":null,"merge_commit_sha":"25c6fcb88ec09ecf92c46b83932b8dc70b2bfe39","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/BD2KGenomics/toil/pulls/769/commits","review_comments_url":"https://api.github.com/repos/BD2KGenomics/toil/pulls/769/comments","review_comment_url":"https://api.github.com/repos/BD2KGenomics/toil/pulls/comments{/number}","comments_url":"https://api.github.com/repos/BD2KGenomics/toil/issues/769/comments","statuses_url":"https://api.github.com/repos/BD2KGenomics/toil/statuses/e2f2dda939286a89e4bf850c82b8a8762ae95dcc","head":{"label":"BD2KGenomics:issues/766-Context-Manager-Follow-Up","ref":"issues/766-Context-Manager-Follow-Up","sha":"e2f2dda939286a89e4bf850c82b8a8762ae95dcc","user":{"login":"BD2KGenomics","id":8701293,"avatar_url":"https://avatars.githubusercontent.com/u/8701293?v=3","gravatar_id":"","url":"https://api.github.com/users/BD2KGenomics","html_url":"https://github.com/BD2KGenomics","followers_url":"https://api.github.com/users/BD2KGenomics/followers","following_url":"https://api.github.com/users/BD2KGenomics/following{/other_user}","gists_url":"https://api.github.com/users/BD2KGenomics/gists{/gist_id}","starred_url":"https://api.github.com/users/BD2KGenomics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/BD2KGenomics/subscriptions","organizations_url":"https://api.github.com/users/BD2KGenomics/orgs","repos_url":"https://api.github.com/users/BD2KGenomics/repos","events_url":"https://api.github.com/users/BD2KGenomics/events{/privacy}","received_events_url":"https://api.github.com/users/BD2KGenomics/received_events","type":"Organization","site_admin":false},"repo":{"id":33148320,"name":"toil","full_name":"BD2KGenomics/toil","owner":{"login":"BD2KGenomics","id":8701293,"avatar_url":"https://avatars.githubusercontent.com/u/8701293?v=3","gravatar_id":"","url":"https://api.github.com/users/BD2KGenomics","html_url":"https://github.com/BD2KGenomics","followers_url":"https://api.github.com/users/BD2KGenomics/followers","following_url":"https://api.github.com/users/BD2KGenomics/following{/other_user}","gists_url":"https://api.github.com/users/BD2KGenomics/gists{/gist_id}","starred_url":"https://api.github.com/users/BD2KGenomics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/BD2KGenomics/subscriptions","organizations_url":"https://api.github.com/users/BD2KGenomics/orgs","repos_url":"https://api.github.com/users/BD2KGenomics/repos","events_url":"https://api.github.com/users/BD2KGenomics/events{/privacy}","received_events_url":"https://api.github.com/users/BD2KGenomics/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/BD2KGenomics/toil","description":"Toil is a scalable, efficient, cross-platform pipeline management system, written entirely in Python, and designed around the principles of functional programming.","fork":false,"url":"https://api.github.com/repos/BD2KGenomics/toil","forks_url":"https://api.github.com/repos/BD2KGenomics/toil/forks","keys_url":"https://api.github.com/repos/BD2KGenomics/toil/keys{/key_id}","collaborators_url":"https://api.github.com/repos/BD2KGenomics/toil/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/BD2KGenomics/toil/teams","hooks_url":"https://api.github.com/repos/BD2KGenomics/toil/hooks","issue_events_url":"https://api.github.com/repos/BD2KGenomics/toil/issues/events{/number}","events_url":"https://api.github.com/repos/BD2KGenomics/toil/events","assignees_url":"https://api.github.com/repos/BD2KGenomics/toil/assignees{/user}","branches_url":"https://api.github.com/repos/BD2KGenomics/toil/branches{/branch}","tags_url":"https://api.github.com/repos/BD2KGenomics/toil/tags","blobs_url":"https://api.github.com/repos/BD2KGenomics/toil/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/BD2KGenomics/toil/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/BD2KGenomics/toil/git/refs{/sha}","trees_url":"https://api.github.com/repos/BD2KGenomics/toil/git/trees{/sha}","statuses_url":"https://api.github.com/repos/BD2KGenomics/toil/statuses/{sha}","languages_url":"https://api.github.com/repos/BD2KGenomics/toil/languages","stargazers_url":"https://api.github.com/repos/BD2KGenomics/toil/stargazers","contributors_url":"https://api.github.com/repos/BD2KGenomics/toil/contributors","subscribers_url":"https://api.github.com/repos/BD2KGenomics/toil/subscribers","subscription_url":"https://api.github.com/repos/BD2KGenomics/toil/subscription","commits_url":"https://api.github.com/repos/BD2KGenomics/toil/commits{/sha}","git_commits_url":"https://api.github.com/repos/BD2KGenomics/toil/git/commits{/sha}","comments_url":"https://api.github.com/repos/BD2KGenomics/toil/comments{/number}","issue_comment_url":"https://api.github.com/repos/BD2KGenomics/toil/issues/comments{/number}","contents_url":"https://api.github.com/repos/BD2KGenomics/toil/contents/{+path}","compare_url":"https://api.github.com/repos/BD2KGenomics/toil/compare/{base}...{head}","merges_url":"https://api.github.com/repos/BD2KGenomics/toil/merges","archive_url":"https://api.github.com/repos/BD2KGenomics/toil/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/BD2KGenomics/toil/downloads","issues_url":"https://api.github.com/repos/BD2KGenomics/toil/issues{/number}","pulls_url":"https://api.github.com/repos/BD2KGenomics/toil/pulls{/number}","milestones_url":"https://api.github.com/repos/BD2KGenomics/toil/milestones{/number}","notifications_url":"https://api.github.com/repos/BD2KGenomics/toil/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/BD2KGenomics/toil/labels{/name}","releases_url":"https://api.github.com/repos/BD2KGenomics/toil/releases{/id}","deployments_url":"https://api.github.com/repos/BD2KGenomics/toil/deployments","created_at":"2015-03-30T21:08:58Z","updated_at":"2016-04-24T16:16:49Z","pushed_at":"2016-04-28T07:03:48Z","git_url":"git://github.com/BD2KGenomics/toil.git","ssh_url":"git@github.com:BD2KGenomics/toil.git","clone_url":"https://github.com/BD2KGenomics/toil.git","svn_url":"https://github.com/BD2KGenomics/toil","homepage":"","size":6202,"stargazers_count":38,"watchers_count":38,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":17,"mirror_url":null,"open_issues_count":160,"forks":17,"open_issues":160,"watchers":38,"default_branch":"master"}},"base":{"label":"BD2KGenomics:master","ref":"master","sha":"14f4b8e793a928741180d389870772ac979770ea","user":{"login":"BD2KGenomics","id":8701293,"avatar_url":"https://avatars.githubusercontent.com/u/8701293?v=3","gravatar_id":"","url":"https://api.github.com/users/BD2KGenomics","html_url":"https://github.com/BD2KGenomics","followers_url":"https://api.github.com/users/BD2KGenomics/followers","following_url":"https://api.github.com/users/BD2KGenomics/following{/other_user}","gists_url":"https://api.github.com/users/BD2KGenomics/gists{/gist_id}","starred_url":"https://api.github.com/users/BD2KGenomics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/BD2KGenomics/subscriptions","organizations_url":"https://api.github.com/users/BD2KGenomics/orgs","repos_url":"https://api.github.com/users/BD2KGenomics/repos","events_url":"https://api.github.com/users/BD2KGenomics/events{/privacy}","received_events_url":"https://api.github.com/users/BD2KGenomics/received_events","type":"Organization","site_admin":false},"repo":{"id":33148320,"name":"toil","full_name":"BD2KGenomics/toil","owner":{"login":"BD2KGenomics","id":8701293,"avatar_url":"https://avatars.githubusercontent.com/u/8701293?v=3","gravatar_id":"","url":"https://api.github.com/users/BD2KGenomics","html_url":"https://github.com/BD2KGenomics","followers_url":"https://api.github.com/users/BD2KGenomics/followers","following_url":"https://api.github.com/users/BD2KGenomics/following{/other_user}","gists_url":"https://api.github.com/users/BD2KGenomics/gists{/gist_id}","starred_url":"https://api.github.com/users/BD2KGenomics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/BD2KGenomics/subscriptions","organizations_url":"https://api.github.com/users/BD2KGenomics/orgs","repos_url":"https://api.github.com/users/BD2KGenomics/repos","events_url":"https://api.github.com/users/BD2KGenomics/events{/privacy}","received_events_url":"https://api.github.com/users/BD2KGenomics/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/BD2KGenomics/toil","description":"Toil is a scalable, efficient, cross-platform pipeline management system, written entirely in Python, and designed around the principles of functional programming.","fork":false,"url":"https://api.github.com/repos/BD2KGenomics/toil","forks_url":"https://api.github.com/repos/BD2KGenomics/toil/forks","keys_url":"https://api.github.com/repos/BD2KGenomics/toil/keys{/key_id}","collaborators_url":"https://api.github.com/repos/BD2KGenomics/toil/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/BD2KGenomics/toil/teams","hooks_url":"https://api.github.com/repos/BD2KGenomics/toil/hooks","issue_events_url":"https://api.github.com/repos/BD2KGenomics/toil/issues/events{/number}","events_url":"https://api.github.com/repos/BD2KGenomics/toil/events","assignees_url":"https://api.github.com/repos/BD2KGenomics/toil/assignees{/user}","branches_url":"https://api.github.com/repos/BD2KGenomics/toil/branches{/branch}","tags_url":"https://api.github.com/repos/BD2KGenomics/toil/tags","blobs_url":"https://api.github.com/repos/BD2KGenomics/toil/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/BD2KGenomics/toil/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/BD2KGenomics/toil/git/refs{/sha}","trees_url":"https://api.github.com/repos/BD2KGenomics/toil/git/trees{/sha}","statuses_url":"https://api.github.com/repos/BD2KGenomics/toil/statuses/{sha}","languages_url":"https://api.github.com/repos/BD2KGenomics/toil/languages","stargazers_url":"https://api.github.com/repos/BD2KGenomics/toil/stargazers","contributors_url":"https://api.github.com/repos/BD2KGenomics/toil/contributors","subscribers_url":"https://api.github.com/repos/BD2KGenomics/toil/subscribers","subscription_url":"https://api.github.com/repos/BD2KGenomics/toil/subscription","commits_url":"https://api.github.com/repos/BD2KGenomics/toil/commits{/sha}","git_commits_url":"https://api.github.com/repos/BD2KGenomics/toil/git/commits{/sha}","comments_url":"https://api.github.com/repos/BD2KGenomics/toil/comments{/number}","issue_comment_url":"https://api.github.com/repos/BD2KGenomics/toil/issues/comments{/number}","contents_url":"https://api.github.com/repos/BD2KGenomics/toil/contents/{+path}","compare_url":"https://api.github.com/repos/BD2KGenomics/toil/compare/{base}...{head}","merges_url":"https://api.github.com/repos/BD2KGenomics/toil/merges","archive_url":"https://api.github.com/repos/BD2KGenomics/toil/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/BD2KGenomics/toil/downloads","issues_url":"https://api.github.com/repos/BD2KGenomics/toil/issues{/number}","pulls_url":"https://api.github.com/repos/BD2KGenomics/toil/pulls{/number}","milestones_url":"https://api.github.com/repos/BD2KGenomics/toil/milestones{/number}","notifications_url":"https://api.github.com/repos/BD2KGenomics/toil/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/BD2KGenomics/toil/labels{/name}","releases_url":"https://api.github.com/repos/BD2KGenomics/toil/releases{/id}","deployments_url":"https://api.github.com/repos/BD2KGenomics/toil/deployments","created_at":"2015-03-30T21:08:58Z","updated_at":"2016-04-24T16:16:49Z","pushed_at":"2016-04-28T07:03:48Z","git_url":"git://github.com/BD2KGenomics/toil.git","ssh_url":"git@github.com:BD2KGenomics/toil.git","clone_url":"https://github.com/BD2KGenomics/toil.git","svn_url":"https://github.com/BD2KGenomics/toil","homepage":"","size":6202,"stargazers_count":38,"watchers_count":38,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":17,"mirror_url":null,"open_issues_count":160,"forks":17,"open_issues":160,"watchers":38,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/BD2KGenomics/toil/pulls/769"},"html":{"href":"https://github.com/BD2KGenomics/toil/pull/769"},"issue":{"href":"https://api.github.com/repos/BD2KGenomics/toil/issues/769"},"comments":{"href":"https://api.github.com/repos/BD2KGenomics/toil/issues/769/comments"},"review_comments":{"href":"https://api.github.com/repos/BD2KGenomics/toil/pulls/769/comments"},"review_comment":{"href":"https://api.github.com/repos/BD2KGenomics/toil/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/BD2KGenomics/toil/pulls/769/commits"},"statuses":{"href":"https://api.github.com/repos/BD2KGenomics/toil/statuses/e2f2dda939286a89e4bf850c82b8a8762ae95dcc"}}}}