{"action":"created","comment":{"url":"https://api.github.com/repos/BD2KGenomics/toil/pulls/comments/67921973","id":67921973,"diff_hunk":"@@ -74,38 +82,70 @@ def copyKeyMultipart(srcKey, dstBucketName, dstKeyName, headers=None):\n     :rtype: boto.s3.multipart.CompletedMultiPartUpload\n     :return: An object representing the completed upload.\n     \"\"\"\n+    def copyPart(partIndex):\n+        if exceptions:\n+            return None\n+        else:\n+            try:\n+                for attempt in retry_s3():\n+                    with attempt:\n+                        # Note that part_num is indexed from 1 while start and end are indexed from\n+                        # 0. Notice also that the partIndex variable is indexed from 0.\n+                        part = upload.copy_part_from_key(src_bucket_name=srcKey.bucket.name,\n+                                                         src_key_name=srcKey.name,\n+                                                         src_version_id=srcKey.version_id,\n+                                                         part_num=partIndex + 1,\n+                                                         start=partIndex * partSize,\n+                                                         end=min(partIndex * partSize + partSize,\n+                                                                 totalSize) - 1,\n+                                                         headers=headers)\n+                log.debug('Successfully copied part %d of %d' % (partIndex, totalParts))\n+\n+            except Exception as e:\n+                if partIndex <= 4:\n+                    exceptions[partIndex] = traceback.format_exc()\n+                else:\n+                    exceptions[partIndex] = e\n+                log.debug('Part number %s of copy failed.', partIndex)\n+                return None\n+            else:\n+                return part\n+\n     partSize = defaultPartSize\n+    totalSize = srcKey.size\n+    totalParts = (totalSize + partSize - 1) / partSize\n+    exceptions = collections.OrderedDict()\n     # We need a location-agnostic connection to S3 so we can't use the one that we\n     # normally use for interacting with the job store bucket.\n     with closing(boto.connect_s3()) as s3:\n-        headers = headers or {}\n-        totalSize = srcKey.size\n         for attempt in retry_s3():\n             with attempt:\n                 dstBucket = s3.get_bucket(dstBucketName)\n                 upload = dstBucket.initiate_multipart_upload(dstKeyName, headers=headers)\n+        log.info(\"Initiated multipart copy from 's3://%s/%s' to 's3://%s/%s'.\" %\n+                 (srcKey.bucket.name, srcKey.name, dstBucketName, dstKeyName))\n         try:\n-            start = 0\n-            partIndex = itertools.count()\n-            while start < totalSize:\n-                end = min(start + partSize, totalSize)\n-                for attempt in retry_s3():\n-                    with attempt:\n-                        upload.copy_part_from_key(src_bucket_name=srcKey.bucket.name,\n-                                                  src_key_name=srcKey.name,\n-                                                  src_version_id=srcKey.version_id,\n-                                                  part_num=next(partIndex) + 1,\n-                                                  start=start,\n-                                                  end=end - 1,\n-                                                  headers=headers)\n-                start += partSize\n+            with ThreadPoolExecutor(max_workers=min(cpu_count() * 16, totalParts, 128)) as executor:\n+                finishedParts = set(executor.map(copyPart, range(0, totalParts)))","path":"src/toil/jobStores/aws/jobStore.py","position":98,"original_position":98,"commit_id":"891f71f7e1e9055c4c4ce1ec0f84225d1a6f3e46","original_commit_id":"891f71f7e1e9055c4c4ce1ec0f84225d1a6f3e46","user":{"login":"hannes-ucsc","id":5143256,"avatar_url":"https://avatars.githubusercontent.com/u/5143256?v=3","gravatar_id":"","url":"https://api.github.com/users/hannes-ucsc","html_url":"https://github.com/hannes-ucsc","followers_url":"https://api.github.com/users/hannes-ucsc/followers","following_url":"https://api.github.com/users/hannes-ucsc/following{/other_user}","gists_url":"https://api.github.com/users/hannes-ucsc/gists{/gist_id}","starred_url":"https://api.github.com/users/hannes-ucsc/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hannes-ucsc/subscriptions","organizations_url":"https://api.github.com/users/hannes-ucsc/orgs","repos_url":"https://api.github.com/users/hannes-ucsc/repos","events_url":"https://api.github.com/users/hannes-ucsc/events{/privacy}","received_events_url":"https://api.github.com/users/hannes-ucsc/received_events","type":"User","site_admin":false},"body":"xrange() is probably better in this case","created_at":"2016-06-21T18:13:09Z","updated_at":"2016-06-21T18:13:09Z","html_url":"https://github.com/BD2KGenomics/toil/pull/867#discussion_r67921973","pull_request_url":"https://api.github.com/repos/BD2KGenomics/toil/pulls/867","_links":{"self":{"href":"https://api.github.com/repos/BD2KGenomics/toil/pulls/comments/67921973"},"html":{"href":"https://github.com/BD2KGenomics/toil/pull/867#discussion_r67921973"},"pull_request":{"href":"https://api.github.com/repos/BD2KGenomics/toil/pulls/867"}}},"pull_request":{"url":"https://api.github.com/repos/BD2KGenomics/toil/pulls/867","id":70068103,"html_url":"https://github.com/BD2KGenomics/toil/pull/867","diff_url":"https://github.com/BD2KGenomics/toil/pull/867.diff","patch_url":"https://github.com/BD2KGenomics/toil/pull/867.patch","issue_url":"https://api.github.com/repos/BD2KGenomics/toil/issues/867","number":867,"state":"open","locked":false,"title":"Multipart copy now uses a thread pool (resolves #859)","user":{"login":"JakeNarkizian","id":10471159,"avatar_url":"https://avatars.githubusercontent.com/u/10471159?v=3","gravatar_id":"","url":"https://api.github.com/users/JakeNarkizian","html_url":"https://github.com/JakeNarkizian","followers_url":"https://api.github.com/users/JakeNarkizian/followers","following_url":"https://api.github.com/users/JakeNarkizian/following{/other_user}","gists_url":"https://api.github.com/users/JakeNarkizian/gists{/gist_id}","starred_url":"https://api.github.com/users/JakeNarkizian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/JakeNarkizian/subscriptions","organizations_url":"https://api.github.com/users/JakeNarkizian/orgs","repos_url":"https://api.github.com/users/JakeNarkizian/repos","events_url":"https://api.github.com/users/JakeNarkizian/events{/privacy}","received_events_url":"https://api.github.com/users/JakeNarkizian/received_events","type":"User","site_admin":false},"body":"Copies within the same region should now be much faster.","created_at":"2016-05-13T22:38:51Z","updated_at":"2016-06-21T18:13:09Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7fdc50c7ab69078f0b6015e2fece59b760627085","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/BD2KGenomics/toil/pulls/867/commits","review_comments_url":"https://api.github.com/repos/BD2KGenomics/toil/pulls/867/comments","review_comment_url":"https://api.github.com/repos/BD2KGenomics/toil/pulls/comments{/number}","comments_url":"https://api.github.com/repos/BD2KGenomics/toil/issues/867/comments","statuses_url":"https://api.github.com/repos/BD2KGenomics/toil/statuses/891f71f7e1e9055c4c4ce1ec0f84225d1a6f3e46","head":{"label":"BD2KGenomics:issues/859-multipart-copy-threadpool","ref":"issues/859-multipart-copy-threadpool","sha":"891f71f7e1e9055c4c4ce1ec0f84225d1a6f3e46","user":{"login":"BD2KGenomics","id":8701293,"avatar_url":"https://avatars.githubusercontent.com/u/8701293?v=3","gravatar_id":"","url":"https://api.github.com/users/BD2KGenomics","html_url":"https://github.com/BD2KGenomics","followers_url":"https://api.github.com/users/BD2KGenomics/followers","following_url":"https://api.github.com/users/BD2KGenomics/following{/other_user}","gists_url":"https://api.github.com/users/BD2KGenomics/gists{/gist_id}","starred_url":"https://api.github.com/users/BD2KGenomics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/BD2KGenomics/subscriptions","organizations_url":"https://api.github.com/users/BD2KGenomics/orgs","repos_url":"https://api.github.com/users/BD2KGenomics/repos","events_url":"https://api.github.com/users/BD2KGenomics/events{/privacy}","received_events_url":"https://api.github.com/users/BD2KGenomics/received_events","type":"Organization","site_admin":false},"repo":{"id":33148320,"name":"toil","full_name":"BD2KGenomics/toil","owner":{"login":"BD2KGenomics","id":8701293,"avatar_url":"https://avatars.githubusercontent.com/u/8701293?v=3","gravatar_id":"","url":"https://api.github.com/users/BD2KGenomics","html_url":"https://github.com/BD2KGenomics","followers_url":"https://api.github.com/users/BD2KGenomics/followers","following_url":"https://api.github.com/users/BD2KGenomics/following{/other_user}","gists_url":"https://api.github.com/users/BD2KGenomics/gists{/gist_id}","starred_url":"https://api.github.com/users/BD2KGenomics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/BD2KGenomics/subscriptions","organizations_url":"https://api.github.com/users/BD2KGenomics/orgs","repos_url":"https://api.github.com/users/BD2KGenomics/repos","events_url":"https://api.github.com/users/BD2KGenomics/events{/privacy}","received_events_url":"https://api.github.com/users/BD2KGenomics/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/BD2KGenomics/toil","description":"A scalable, efficient, cross-platform and easy-to-use workflow engine in pure Python","fork":false,"url":"https://api.github.com/repos/BD2KGenomics/toil","forks_url":"https://api.github.com/repos/BD2KGenomics/toil/forks","keys_url":"https://api.github.com/repos/BD2KGenomics/toil/keys{/key_id}","collaborators_url":"https://api.github.com/repos/BD2KGenomics/toil/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/BD2KGenomics/toil/teams","hooks_url":"https://api.github.com/repos/BD2KGenomics/toil/hooks","issue_events_url":"https://api.github.com/repos/BD2KGenomics/toil/issues/events{/number}","events_url":"https://api.github.com/repos/BD2KGenomics/toil/events","assignees_url":"https://api.github.com/repos/BD2KGenomics/toil/assignees{/user}","branches_url":"https://api.github.com/repos/BD2KGenomics/toil/branches{/branch}","tags_url":"https://api.github.com/repos/BD2KGenomics/toil/tags","blobs_url":"https://api.github.com/repos/BD2KGenomics/toil/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/BD2KGenomics/toil/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/BD2KGenomics/toil/git/refs{/sha}","trees_url":"https://api.github.com/repos/BD2KGenomics/toil/git/trees{/sha}","statuses_url":"https://api.github.com/repos/BD2KGenomics/toil/statuses/{sha}","languages_url":"https://api.github.com/repos/BD2KGenomics/toil/languages","stargazers_url":"https://api.github.com/repos/BD2KGenomics/toil/stargazers","contributors_url":"https://api.github.com/repos/BD2KGenomics/toil/contributors","subscribers_url":"https://api.github.com/repos/BD2KGenomics/toil/subscribers","subscription_url":"https://api.github.com/repos/BD2KGenomics/toil/subscription","commits_url":"https://api.github.com/repos/BD2KGenomics/toil/commits{/sha}","git_commits_url":"https://api.github.com/repos/BD2KGenomics/toil/git/commits{/sha}","comments_url":"https://api.github.com/repos/BD2KGenomics/toil/comments{/number}","issue_comment_url":"https://api.github.com/repos/BD2KGenomics/toil/issues/comments{/number}","contents_url":"https://api.github.com/repos/BD2KGenomics/toil/contents/{+path}","compare_url":"https://api.github.com/repos/BD2KGenomics/toil/compare/{base}...{head}","merges_url":"https://api.github.com/repos/BD2KGenomics/toil/merges","archive_url":"https://api.github.com/repos/BD2KGenomics/toil/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/BD2KGenomics/toil/downloads","issues_url":"https://api.github.com/repos/BD2KGenomics/toil/issues{/number}","pulls_url":"https://api.github.com/repos/BD2KGenomics/toil/pulls{/number}","milestones_url":"https://api.github.com/repos/BD2KGenomics/toil/milestones{/number}","notifications_url":"https://api.github.com/repos/BD2KGenomics/toil/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/BD2KGenomics/toil/labels{/name}","releases_url":"https://api.github.com/repos/BD2KGenomics/toil/releases{/id}","deployments_url":"https://api.github.com/repos/BD2KGenomics/toil/deployments","created_at":"2015-03-30T21:08:58Z","updated_at":"2016-06-14T17:41:39Z","pushed_at":"2016-06-21T09:47:20Z","git_url":"git://github.com/BD2KGenomics/toil.git","ssh_url":"git@github.com:BD2KGenomics/toil.git","clone_url":"https://github.com/BD2KGenomics/toil.git","svn_url":"https://github.com/BD2KGenomics/toil","homepage":"","size":6647,"stargazers_count":190,"watchers_count":190,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":33,"mirror_url":null,"open_issues_count":139,"forks":33,"open_issues":139,"watchers":190,"default_branch":"master"}},"base":{"label":"BD2KGenomics:master","ref":"master","sha":"2d79007b91d0c9bc658035527379d25a0e28d094","user":{"login":"BD2KGenomics","id":8701293,"avatar_url":"https://avatars.githubusercontent.com/u/8701293?v=3","gravatar_id":"","url":"https://api.github.com/users/BD2KGenomics","html_url":"https://github.com/BD2KGenomics","followers_url":"https://api.github.com/users/BD2KGenomics/followers","following_url":"https://api.github.com/users/BD2KGenomics/following{/other_user}","gists_url":"https://api.github.com/users/BD2KGenomics/gists{/gist_id}","starred_url":"https://api.github.com/users/BD2KGenomics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/BD2KGenomics/subscriptions","organizations_url":"https://api.github.com/users/BD2KGenomics/orgs","repos_url":"https://api.github.com/users/BD2KGenomics/repos","events_url":"https://api.github.com/users/BD2KGenomics/events{/privacy}","received_events_url":"https://api.github.com/users/BD2KGenomics/received_events","type":"Organization","site_admin":false},"repo":{"id":33148320,"name":"toil","full_name":"BD2KGenomics/toil","owner":{"login":"BD2KGenomics","id":8701293,"avatar_url":"https://avatars.githubusercontent.com/u/8701293?v=3","gravatar_id":"","url":"https://api.github.com/users/BD2KGenomics","html_url":"https://github.com/BD2KGenomics","followers_url":"https://api.github.com/users/BD2KGenomics/followers","following_url":"https://api.github.com/users/BD2KGenomics/following{/other_user}","gists_url":"https://api.github.com/users/BD2KGenomics/gists{/gist_id}","starred_url":"https://api.github.com/users/BD2KGenomics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/BD2KGenomics/subscriptions","organizations_url":"https://api.github.com/users/BD2KGenomics/orgs","repos_url":"https://api.github.com/users/BD2KGenomics/repos","events_url":"https://api.github.com/users/BD2KGenomics/events{/privacy}","received_events_url":"https://api.github.com/users/BD2KGenomics/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/BD2KGenomics/toil","description":"A scalable, efficient, cross-platform and easy-to-use workflow engine in pure Python","fork":false,"url":"https://api.github.com/repos/BD2KGenomics/toil","forks_url":"https://api.github.com/repos/BD2KGenomics/toil/forks","keys_url":"https://api.github.com/repos/BD2KGenomics/toil/keys{/key_id}","collaborators_url":"https://api.github.com/repos/BD2KGenomics/toil/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/BD2KGenomics/toil/teams","hooks_url":"https://api.github.com/repos/BD2KGenomics/toil/hooks","issue_events_url":"https://api.github.com/repos/BD2KGenomics/toil/issues/events{/number}","events_url":"https://api.github.com/repos/BD2KGenomics/toil/events","assignees_url":"https://api.github.com/repos/BD2KGenomics/toil/assignees{/user}","branches_url":"https://api.github.com/repos/BD2KGenomics/toil/branches{/branch}","tags_url":"https://api.github.com/repos/BD2KGenomics/toil/tags","blobs_url":"https://api.github.com/repos/BD2KGenomics/toil/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/BD2KGenomics/toil/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/BD2KGenomics/toil/git/refs{/sha}","trees_url":"https://api.github.com/repos/BD2KGenomics/toil/git/trees{/sha}","statuses_url":"https://api.github.com/repos/BD2KGenomics/toil/statuses/{sha}","languages_url":"https://api.github.com/repos/BD2KGenomics/toil/languages","stargazers_url":"https://api.github.com/repos/BD2KGenomics/toil/stargazers","contributors_url":"https://api.github.com/repos/BD2KGenomics/toil/contributors","subscribers_url":"https://api.github.com/repos/BD2KGenomics/toil/subscribers","subscription_url":"https://api.github.com/repos/BD2KGenomics/toil/subscription","commits_url":"https://api.github.com/repos/BD2KGenomics/toil/commits{/sha}","git_commits_url":"https://api.github.com/repos/BD2KGenomics/toil/git/commits{/sha}","comments_url":"https://api.github.com/repos/BD2KGenomics/toil/comments{/number}","issue_comment_url":"https://api.github.com/repos/BD2KGenomics/toil/issues/comments{/number}","contents_url":"https://api.github.com/repos/BD2KGenomics/toil/contents/{+path}","compare_url":"https://api.github.com/repos/BD2KGenomics/toil/compare/{base}...{head}","merges_url":"https://api.github.com/repos/BD2KGenomics/toil/merges","archive_url":"https://api.github.com/repos/BD2KGenomics/toil/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/BD2KGenomics/toil/downloads","issues_url":"https://api.github.com/repos/BD2KGenomics/toil/issues{/number}","pulls_url":"https://api.github.com/repos/BD2KGenomics/toil/pulls{/number}","milestones_url":"https://api.github.com/repos/BD2KGenomics/toil/milestones{/number}","notifications_url":"https://api.github.com/repos/BD2KGenomics/toil/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/BD2KGenomics/toil/labels{/name}","releases_url":"https://api.github.com/repos/BD2KGenomics/toil/releases{/id}","deployments_url":"https://api.github.com/repos/BD2KGenomics/toil/deployments","created_at":"2015-03-30T21:08:58Z","updated_at":"2016-06-14T17:41:39Z","pushed_at":"2016-06-21T09:47:20Z","git_url":"git://github.com/BD2KGenomics/toil.git","ssh_url":"git@github.com:BD2KGenomics/toil.git","clone_url":"https://github.com/BD2KGenomics/toil.git","svn_url":"https://github.com/BD2KGenomics/toil","homepage":"","size":6647,"stargazers_count":190,"watchers_count":190,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":33,"mirror_url":null,"open_issues_count":139,"forks":33,"open_issues":139,"watchers":190,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/BD2KGenomics/toil/pulls/867"},"html":{"href":"https://github.com/BD2KGenomics/toil/pull/867"},"issue":{"href":"https://api.github.com/repos/BD2KGenomics/toil/issues/867"},"comments":{"href":"https://api.github.com/repos/BD2KGenomics/toil/issues/867/comments"},"review_comments":{"href":"https://api.github.com/repos/BD2KGenomics/toil/pulls/867/comments"},"review_comment":{"href":"https://api.github.com/repos/BD2KGenomics/toil/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/BD2KGenomics/toil/pulls/867/commits"},"statuses":{"href":"https://api.github.com/repos/BD2KGenomics/toil/statuses/891f71f7e1e9055c4c4ce1ec0f84225d1a6f3e46"}}}}
{"actor":{"display_login":"hannes-ucsc"}}