{"action":"created","comment":{"url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/comments/78884992","id":78884992,"diff_hunk":"@@ -0,0 +1,368 @@\n+# -*- coding: utf-8 -*-\n+# coding=utf-8\n+# --------------------------------------------------------------------------\n+# Copyright (c) Microsoft Corporation. All rights reserved.\n+# Licensed under the MIT License. See License.txt in the project root for\n+# license information.\n+# --------------------------------------------------------------------------\n+\n+\"\"\"\n+High performance multi-threaded module to up/download\n+\n+Calls method in `core` with thread pool executor to ensure the network\n+is used to its maximum throughput.\n+\n+Only implements upload and download of (massive) files and directory trees.\n+\"\"\"\n+from collections import namedtuple\n+from concurrent.futures import ThreadPoolExecutor\n+import logging\n+import multiprocessing\n+import os\n+import pickle\n+import threading\n+import time\n+import uuid\n+\n+from adlfs.utils import clamp\n+\n+MAX_THREADS = 64\n+\n+logger = logging.getLogger(__name__)\n+\n+File = namedtuple('File', 'src dst state nbytes start stop chunks')\n+Chunk = namedtuple('Chunk', 'name state offset retries')\n+\n+\n+class DisjointState(object):\n+    \"\"\"\n+    Collection of disjoint sets containing state about objects.\n+\n+    Internal class used by `ADLTransferClient`.\n+    \"\"\"\n+    def __init__(self, *states):\n+        self._states = {state: set() for state in states}\n+        self._objects = {}\n+\n+    @property\n+    def states(self):\n+        return list(self._states)\n+\n+    @property\n+    def objects(self):\n+        return list(self._objects)\n+\n+    def __iter__(self):\n+        return iter(self._objects.items())\n+\n+    def __getitem__(self, obj):\n+        return self._objects[obj]\n+\n+    def __setitem__(self, obj, state):\n+        if obj in self._objects:\n+            self._states[self._objects[obj]].discard(obj)\n+        self._states[state].add(obj)\n+        self._objects[obj] = state\n+\n+    def contains_all(self, state):\n+        objs = self._states[state]\n+        return len(objs) > 0 and len(self.objects) - len(objs) == 0\n+\n+    def contains_none(self, *states):\n+        return all([len(self._states[state]) == 0 for state in states])\n+\n+    def __str__(self):\n+        status = \" \".join(\n+            [\"%s=%d\" % (s, len(self._states[s])) for s in self._states])\n+        return \"<DisjointState: \" + status + \">\"\n+\n+    __repr__ = __str__\n+\n+\n+class ADLTransferClient(object):\n+    \"\"\"\n+    Client for transferring data from/to Azure DataLake Store\n+\n+    Parameters\n+    ----------\n+    adlfs: ADL filesystem instance\n+    name: str\n+        Unique ID used for persistence.\n+    transfer: callable\n+        Function or callable object invoked when transferring chunks. See\n+        ``Function Signatures``.\n+    merge: callable [None]\n+        Function or callable object invoked when merging chunks. If None,\n+        then merging is skipped. See ``Function Signatures``.","path":"adlfs/transfer.py","position":96,"original_position":96,"commit_id":"aecb1744f895e63c92c0d72373fc15f396aad6a2","original_commit_id":"aecb1744f895e63c92c0d72373fc15f396aad6a2","user":{"login":"jbcrail","id":6038,"avatar_url":"https://avatars.githubusercontent.com/u/6038?v=3","gravatar_id":"","url":"https://api.github.com/users/jbcrail","html_url":"https://github.com/jbcrail","followers_url":"https://api.github.com/users/jbcrail/followers","following_url":"https://api.github.com/users/jbcrail/following{/other_user}","gists_url":"https://api.github.com/users/jbcrail/gists{/gist_id}","starred_url":"https://api.github.com/users/jbcrail/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbcrail/subscriptions","organizations_url":"https://api.github.com/users/jbcrail/orgs","repos_url":"https://api.github.com/users/jbcrail/repos","events_url":"https://api.github.com/users/jbcrail/events{/privacy}","received_events_url":"https://api.github.com/users/jbcrail/received_events","type":"User","site_admin":false},"body":"In the case where we are transferring only one small file or one chunk, then we disable the merge step and do not write to a temporary file.","created_at":"2016-09-15T03:34:38Z","updated_at":"2016-09-15T03:34:38Z","html_url":"https://github.com/Azure/azure-data-lake-store-python/pull/64#discussion_r78884992","pull_request_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/64","_links":{"self":{"href":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/comments/78884992"},"html":{"href":"https://github.com/Azure/azure-data-lake-store-python/pull/64#discussion_r78884992"},"pull_request":{"href":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/64"}}},"pull_request":{"url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/64","id":85134562,"html_url":"https://github.com/Azure/azure-data-lake-store-python/pull/64","diff_url":"https://github.com/Azure/azure-data-lake-store-python/pull/64.diff","patch_url":"https://github.com/Azure/azure-data-lake-store-python/pull/64.patch","issue_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/issues/64","number":64,"state":"open","locked":false,"title":"PR: Downloader/uploader improvements","user":{"login":"jbcrail","id":6038,"avatar_url":"https://avatars.githubusercontent.com/u/6038?v=3","gravatar_id":"","url":"https://api.github.com/users/jbcrail","html_url":"https://github.com/jbcrail","followers_url":"https://api.github.com/users/jbcrail/followers","following_url":"https://api.github.com/users/jbcrail/following{/other_user}","gists_url":"https://api.github.com/users/jbcrail/gists{/gist_id}","starred_url":"https://api.github.com/users/jbcrail/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbcrail/subscriptions","organizations_url":"https://api.github.com/users/jbcrail/orgs","repos_url":"https://api.github.com/users/jbcrail/repos","events_url":"https://api.github.com/users/jbcrail/events{/privacy}","received_events_url":"https://api.github.com/users/jbcrail/received_events","type":"User","site_admin":false},"body":"This PR will give us improved control of transfer activity and will allow us to save/resume/monitor with more detail. The common transfer functionality has been refactored into a separate module.\r\n\r\nWhether uploading or downloading, files can be written into temporary files and merged at the end. Previously, this was only available when uploading.\r\n\r\nThe actual callables used to transfer and merge can be overridden, making it easier to add unit tests. For example, the transfer/merge callables can be no-ops, thus, testing core functionality without network traffic.\r\n \r\nOther miscellaneous improvements:\r\n* Add fine-grained states for file and chunk statuses\r\n* Add hooks to view transfer progress (helps with #39)\r\n* Add parameters to tune temporary files\r\n* Add timeout to transfer monitor\r\n* Shutdown transfer more cleanly\r\n* Limit the number of threads (`1 <= n <= MAX_THREADS`)\r\n\r\n~~Note: persistence is currently broken and will be fixed soon.~~","created_at":"2016-09-13T22:08:16Z","updated_at":"2016-09-15T03:34:38Z","closed_at":null,"merged_at":null,"merge_commit_sha":"0349d1d9066be90038bf4396f2568b47abf33d47","assignee":{"login":"martindurant","id":6042212,"avatar_url":"https://avatars.githubusercontent.com/u/6042212?v=3","gravatar_id":"","url":"https://api.github.com/users/martindurant","html_url":"https://github.com/martindurant","followers_url":"https://api.github.com/users/martindurant/followers","following_url":"https://api.github.com/users/martindurant/following{/other_user}","gists_url":"https://api.github.com/users/martindurant/gists{/gist_id}","starred_url":"https://api.github.com/users/martindurant/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/martindurant/subscriptions","organizations_url":"https://api.github.com/users/martindurant/orgs","repos_url":"https://api.github.com/users/martindurant/repos","events_url":"https://api.github.com/users/martindurant/events{/privacy}","received_events_url":"https://api.github.com/users/martindurant/received_events","type":"User","site_admin":false},"assignees":[{"login":"martindurant","id":6042212,"avatar_url":"https://avatars.githubusercontent.com/u/6042212?v=3","gravatar_id":"","url":"https://api.github.com/users/martindurant","html_url":"https://github.com/martindurant","followers_url":"https://api.github.com/users/martindurant/followers","following_url":"https://api.github.com/users/martindurant/following{/other_user}","gists_url":"https://api.github.com/users/martindurant/gists{/gist_id}","starred_url":"https://api.github.com/users/martindurant/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/martindurant/subscriptions","organizations_url":"https://api.github.com/users/martindurant/orgs","repos_url":"https://api.github.com/users/martindurant/repos","events_url":"https://api.github.com/users/martindurant/events{/privacy}","received_events_url":"https://api.github.com/users/martindurant/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/64/commits","review_comments_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/64/comments","review_comment_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/comments{/number}","comments_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/issues/64/comments","statuses_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/statuses/aecb1744f895e63c92c0d72373fc15f396aad6a2","head":{"label":"jbcrail:refactor-transfer","ref":"refactor-transfer","sha":"aecb1744f895e63c92c0d72373fc15f396aad6a2","user":{"login":"jbcrail","id":6038,"avatar_url":"https://avatars.githubusercontent.com/u/6038?v=3","gravatar_id":"","url":"https://api.github.com/users/jbcrail","html_url":"https://github.com/jbcrail","followers_url":"https://api.github.com/users/jbcrail/followers","following_url":"https://api.github.com/users/jbcrail/following{/other_user}","gists_url":"https://api.github.com/users/jbcrail/gists{/gist_id}","starred_url":"https://api.github.com/users/jbcrail/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbcrail/subscriptions","organizations_url":"https://api.github.com/users/jbcrail/orgs","repos_url":"https://api.github.com/users/jbcrail/repos","events_url":"https://api.github.com/users/jbcrail/events{/privacy}","received_events_url":"https://api.github.com/users/jbcrail/received_events","type":"User","site_admin":false},"repo":{"id":65852329,"name":"azure-data-lake-store-python","full_name":"jbcrail/azure-data-lake-store-python","owner":{"login":"jbcrail","id":6038,"avatar_url":"https://avatars.githubusercontent.com/u/6038?v=3","gravatar_id":"","url":"https://api.github.com/users/jbcrail","html_url":"https://github.com/jbcrail","followers_url":"https://api.github.com/users/jbcrail/followers","following_url":"https://api.github.com/users/jbcrail/following{/other_user}","gists_url":"https://api.github.com/users/jbcrail/gists{/gist_id}","starred_url":"https://api.github.com/users/jbcrail/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbcrail/subscriptions","organizations_url":"https://api.github.com/users/jbcrail/orgs","repos_url":"https://api.github.com/users/jbcrail/repos","events_url":"https://api.github.com/users/jbcrail/events{/privacy}","received_events_url":"https://api.github.com/users/jbcrail/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/jbcrail/azure-data-lake-store-python","description":"Microsoft Azure Data Lake Store Filesystem Library for Python","fork":true,"url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python","forks_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/forks","keys_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/teams","hooks_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/hooks","issue_events_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/issues/events{/number}","events_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/events","assignees_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/assignees{/user}","branches_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/branches{/branch}","tags_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/tags","blobs_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/git/refs{/sha}","trees_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/statuses/{sha}","languages_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/languages","stargazers_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/stargazers","contributors_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/contributors","subscribers_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/subscribers","subscription_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/subscription","commits_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/commits{/sha}","git_commits_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/git/commits{/sha}","comments_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/comments{/number}","issue_comment_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/issues/comments{/number}","contents_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/contents/{+path}","compare_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/merges","archive_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/downloads","issues_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/issues{/number}","pulls_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/pulls{/number}","milestones_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/milestones{/number}","notifications_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/labels{/name}","releases_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/releases{/id}","deployments_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/deployments","created_at":"2016-08-16T20:43:34Z","updated_at":"2016-08-16T20:05:18Z","pushed_at":"2016-09-14T21:39:51Z","git_url":"git://github.com/jbcrail/azure-data-lake-store-python.git","ssh_url":"git@github.com:jbcrail/azure-data-lake-store-python.git","clone_url":"https://github.com/jbcrail/azure-data-lake-store-python.git","svn_url":"https://github.com/jbcrail/azure-data-lake-store-python","homepage":null,"size":790,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"Azure:dev","ref":"dev","sha":"50cdb8c89599fffc8bf221ec0b671ac09d39fe32","user":{"login":"Azure","id":6844498,"avatar_url":"https://avatars.githubusercontent.com/u/6844498?v=3","gravatar_id":"","url":"https://api.github.com/users/Azure","html_url":"https://github.com/Azure","followers_url":"https://api.github.com/users/Azure/followers","following_url":"https://api.github.com/users/Azure/following{/other_user}","gists_url":"https://api.github.com/users/Azure/gists{/gist_id}","starred_url":"https://api.github.com/users/Azure/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Azure/subscriptions","organizations_url":"https://api.github.com/users/Azure/orgs","repos_url":"https://api.github.com/users/Azure/repos","events_url":"https://api.github.com/users/Azure/events{/privacy}","received_events_url":"https://api.github.com/users/Azure/received_events","type":"Organization","site_admin":false},"repo":{"id":62902455,"name":"azure-data-lake-store-python","full_name":"Azure/azure-data-lake-store-python","owner":{"login":"Azure","id":6844498,"avatar_url":"https://avatars.githubusercontent.com/u/6844498?v=3","gravatar_id":"","url":"https://api.github.com/users/Azure","html_url":"https://github.com/Azure","followers_url":"https://api.github.com/users/Azure/followers","following_url":"https://api.github.com/users/Azure/following{/other_user}","gists_url":"https://api.github.com/users/Azure/gists{/gist_id}","starred_url":"https://api.github.com/users/Azure/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Azure/subscriptions","organizations_url":"https://api.github.com/users/Azure/orgs","repos_url":"https://api.github.com/users/Azure/repos","events_url":"https://api.github.com/users/Azure/events{/privacy}","received_events_url":"https://api.github.com/users/Azure/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Azure/azure-data-lake-store-python","description":"Microsoft Azure Data Lake Store Filesystem Library for Python","fork":false,"url":"https://api.github.com/repos/Azure/azure-data-lake-store-python","forks_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/forks","keys_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/teams","hooks_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/hooks","issue_events_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/issues/events{/number}","events_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/events","assignees_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/assignees{/user}","branches_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/branches{/branch}","tags_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/tags","blobs_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/git/refs{/sha}","trees_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/statuses/{sha}","languages_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/languages","stargazers_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/stargazers","contributors_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/contributors","subscribers_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/subscribers","subscription_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/subscription","commits_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/commits{/sha}","git_commits_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/git/commits{/sha}","comments_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/comments{/number}","issue_comment_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/issues/comments{/number}","contents_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/contents/{+path}","compare_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/merges","archive_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/downloads","issues_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/issues{/number}","pulls_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls{/number}","milestones_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/milestones{/number}","notifications_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/labels{/name}","releases_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/releases{/id}","deployments_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/deployments","created_at":"2016-07-08T16:31:32Z","updated_at":"2016-08-16T20:05:18Z","pushed_at":"2016-09-14T21:39:51Z","git_url":"git://github.com/Azure/azure-data-lake-store-python.git","ssh_url":"git@github.com:Azure/azure-data-lake-store-python.git","clone_url":"https://github.com/Azure/azure-data-lake-store-python.git","svn_url":"https://github.com/Azure/azure-data-lake-store-python","homepage":null,"size":1341,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":23,"forks":1,"open_issues":23,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/64"},"html":{"href":"https://github.com/Azure/azure-data-lake-store-python/pull/64"},"issue":{"href":"https://api.github.com/repos/Azure/azure-data-lake-store-python/issues/64"},"comments":{"href":"https://api.github.com/repos/Azure/azure-data-lake-store-python/issues/64/comments"},"review_comments":{"href":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/64/comments"},"review_comment":{"href":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/64/commits"},"statuses":{"href":"https://api.github.com/repos/Azure/azure-data-lake-store-python/statuses/aecb1744f895e63c92c0d72373fc15f396aad6a2"}}}}
{"actor":{"display_login":"jbcrail"}}