{"action":"created","comment":{"url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/comments/78792200","id":78792200,"diff_hunk":"@@ -0,0 +1,312 @@\n+# -*- coding: utf-8 -*-\n+# coding=utf-8\n+# --------------------------------------------------------------------------\n+# Copyright (c) Microsoft Corporation. All rights reserved.\n+# Licensed under the MIT License. See License.txt in the project root for\n+# license information.\n+# --------------------------------------------------------------------------\n+\n+\"\"\"\n+High performance multi-threaded module to up/download\n+\n+Calls method in `core` with thread pool executor to ensure the network\n+is used to its maximum throughput.\n+\n+Only implements upload and download of (massive) files and directory trees.\n+\"\"\"\n+from collections import namedtuple\n+from concurrent.futures import ThreadPoolExecutor\n+import logging\n+import multiprocessing\n+import os\n+import pickle\n+import threading\n+import time\n+import uuid\n+\n+logger = logging.getLogger(__name__)\n+\n+File = namedtuple('File', 'src dst state nbytes start stop chunks')\n+Chunk = namedtuple('Chunk', 'name state offset retries')\n+\n+\n+class DisjointState(object):\n+    \"\"\"\n+    Collection of disjoint sets containing state about objects.\n+    \"\"\"\n+    def __init__(self, *states):\n+        self._states = {state: set() for state in states}\n+        self._objects = {}\n+\n+    @property\n+    def states(self):\n+        return list(self._states)\n+\n+    @property\n+    def objects(self):\n+        return list(self._objects)\n+\n+    def __iter__(self):\n+        return iter(self._objects.items())\n+\n+    def __getitem__(self, obj):\n+        return self._objects[obj]\n+\n+    def __setitem__(self, obj, state):\n+        if obj in self._objects:\n+            self._states[self._objects[obj]].discard(obj)\n+        self._states[state].add(obj)\n+        self._objects[obj] = state\n+\n+    def contains_all(self, state):\n+        objs = self._states[state]\n+        return len(objs) > 0 and len(self.objects) - len(objs) == 0\n+\n+    def contains_none(self, *states):\n+        return all([len(self._states[state]) == 0 for state in states])\n+\n+    def __str__(self):\n+        status = \" \".join(\n+            [\"%s=%d\" % (s, len(self._states[s])) for s in self._states])\n+        return \"<DisjointState: \" + status + \">\"\n+\n+    __repr__ = __str__\n+\n+\n+class ADLTransferClient(object):\n+    def __init__(self, adlfs, name, transfer, merge=None, nthreads=None,\n+                 chunksize=2**26, tmp_path='/tmp', tmp_prefix='part_',\n+                 tmp_unique=True, persist_path=None, delimiter=None):\n+        \"\"\"\n+        Parameters\n+        ----------\n+        adlfs: ADL filesystem instance\n+        name: str\n+            Unique ID used for persistence.\n+        transfer: callable\n+            Function or callable object invoked when transferring chunks.\n+        merge: callable [None]\n+            Function or callable object invoked when merging chunks. If None,\n+            then merging is skipped.\n+        nthreads: int [None]\n+            Number of threads to use. If None, uses the number of cores.\n+        chunksize: int [2**26]\n+            Number of bytes in each chunk for splitting big files. Files smaller\n+            than this number will always be transferred in a single thread.\n+        tmp_path: str ['/tmp']\n+            Path used for temporarily storing transferred chunks until chunks\n+            are gathered into a single file. If None, then each chunk will be\n+            written into the same file.\n+        tmp_prefix: str ['part_']\n+            If given and not None, this is used to provide a prefix to the\n+            temporary chunk.\n+        tmp_unique: bool [True]\n+            If True, then a unique ID will be generated to create a subdirectory\n+            containing the temporary chunks. Otherwise, all temporary chunks\n+            will be placed in `tmp_path`.\n+        persist_path: str [None]\n+            Path used for persisting a client's state. If None, then `save()`\n+            and `load()` will be empty operations.\n+        delimiter: byte(s) or None\n+            If set, will transfer blocks using delimiters, as well as split\n+            files for transferring on that delimiter.\n+        \"\"\"\n+        self._adlfs = adlfs\n+        self._name = name\n+        self._transfer = transfer\n+        self._merge = merge\n+        self._nthreads = nthreads or multiprocessing.cpu_count()\n+        self._chunksize = chunksize\n+        self._chunkretries = 5\n+        self._tmp_path = tmp_path\n+        self._tmp_prefix = tmp_prefix\n+        self._tmp_unique = tmp_unique\n+        self._persist_path = persist_path\n+        self._pool = ThreadPoolExecutor(self._nthreads)\n+        self._shutdown_event = threading.Event()\n+        self._files = {}\n+        self._fstates = DisjointState(\n+            'pending', 'transferring', 'merging', 'finished', 'cancelled',\n+            'errored')\n+\n+    def submit(self, src, dst, nbytes):\n+        \"\"\"\n+        All submitted files start in the `pending` state until `run()` is\n+        called.\n+        \"\"\"\n+        self._fstates[(src, dst)] = 'pending'\n+        self._files[(src, dst)] = dict(\n+            nbytes=nbytes,\n+            start=None,\n+            stop=None,\n+            chunks={},\n+            cstates=DisjointState('running', 'finished', 'cancelled', 'errored'),\n+            merge=None)\n+\n+    def _submit(self, fn, *args, **kwargs):\n+        kwargs['shutdown_event'] = self._shutdown_event\n+        return self._pool.submit(fn, *args, **kwargs)\n+\n+    def _scatter(self, src, dst, transfer):\n+        \"\"\" Split a given file into chunks \"\"\"\n+        dic = self._files[(src, dst)]\n+        self._fstates[(src, dst)] = 'transferring'\n+        offsets = list(range(0, dic['nbytes'], self._chunksize))\n+        for offset in offsets:\n+            if self._tmp_path and len(offsets) > 1:\n+                subdir = uuid.uuid1().hex[:10] if self._tmp_unique else ''\n+                prefix = self._tmp_prefix or ''\n+                name = os.path.join(\n+                    self._tmp_path,\n+                    subdir,\n+                    prefix + str(offset))\n+            else:\n+                name = dst\n+            logger.debug(\"Submitted %s, byte offset %d\", name, offset)\n+            dic['cstates'][name] = 'running'\n+            dic['chunks'][name] = dict(\n+                future=self._submit(transfer, self._adlfs, src, name, offset,\n+                                    self._chunksize),\n+                retries=self._chunkretries,\n+                offset=offset)\n+\n+    @property\n+    def progress(self):\n+        files = []\n+        for key in self._files:\n+            src, dst = key\n+            chunks = []\n+            for name in self._files[key]['chunks']:\n+                chunks.append(Chunk(\n+                    name=name,\n+                    state=self._files[key]['cstates'][name],\n+                    offset=self._files[key]['chunks'][name]['offset'],\n+                    retries=self._files[key]['chunks'][name]['retries']))\n+            files.append(File(\n+                src=src,\n+                dst=dst,\n+                state=self._fstates[key],\n+                nbytes=self._files[key]['nbytes'],\n+                start=self._files[key]['start'],\n+                stop=self._files[key]['stop'],\n+                chunks=chunks))\n+        return files\n+\n+    def _status(self, src, dst, nbytes, start, stop):\n+        elapsed = stop - start\n+        rate = nbytes / elapsed / 1024 / 1024\n+        logger.info(\"Transferred %s -> %s in %f seconds at %f MB/s\",\n+                    src, dst, elapsed, rate)\n+\n+    def _update(self):\n+        for (src, dst), dic in self._files.items():\n+            if self._fstates[(src, dst)] == 'transferring':\n+                for name in list(dic['chunks']):\n+                    future = dic['chunks'][name]['future']\n+                    if not future.done():\n+                        continue\n+                    if future.cancelled():\n+                        dic['cstates'][name] = 'cancelled'\n+                    elif future.exception():\n+                        dic['cstates'][name] = 'errored'\n+                    else:\n+                        dic['cstates'][name] = 'finished'\n+                if dic['cstates'].contains_all('finished'):\n+                    logger.debug(\"Chunks transferred\")\n+                    chunks = list(dic['chunks'])\n+                    if self._merge and len(chunks) > 1:\n+                        logger.debug(\"Merging file: %s\", self._fstates[(src, dst)])\n+                        self._fstates[(src, dst)] = 'merging'\n+                        dic['merge'] = self._submit(self._merge, dst, chunks)\n+                    else:\n+                        dic['stop'] = time.time()\n+                        self._fstates[(src, dst)] = 'finished'\n+                        self._status(src, dst, dic['nbytes'], dic['start'], dic['stop'])\n+                elif dic['cstates'].contains_none('running'):\n+                    logger.debug(\"Transfer failed: %s\", dic['cstates'])\n+                    self._fstates[(src, dst)] = 'errored'\n+                else:\n+                    logger.debug(\"Transferring chunks: %s\", dic['cstates'])\n+            elif self._fstates[(src, dst)] == 'merging':\n+                future = dic['merge']\n+                if not future.done():\n+                    continue\n+                if future.cancelled():\n+                    self._fstates[(src, dst)] = 'cancelled'\n+                elif future.exception():\n+                    self._fstates[(src, dst)] = 'errored'\n+                else:\n+                    dic['stop'] = time.time()\n+                    self._fstates[(src, dst)] = 'finished'\n+                    self._status(src, dst, dic['nbytes'], dic['start'], dic['stop'])\n+        self.save()\n+\n+    def run(self, nthreads=None, monitor=True, before_scatter=None):\n+        self._nthreads = nthreads or self._nthreads\n+        for src, dst in self._files:\n+            self._files[(src, dst)]['start'] = time.time()\n+            self._fstates[(src, dst)] = 'transferring'","path":"adlfs/transfer.py","position":248,"original_position":248,"commit_id":"dc06662b7c3099414d37f0667ef121d26803198d","original_commit_id":"dc06662b7c3099414d37f0667ef121d26803198d","user":{"login":"martindurant","id":6042212,"avatar_url":"https://avatars.githubusercontent.com/u/6042212?v=3","gravatar_id":"","url":"https://api.github.com/users/martindurant","html_url":"https://github.com/martindurant","followers_url":"https://api.github.com/users/martindurant/followers","following_url":"https://api.github.com/users/martindurant/following{/other_user}","gists_url":"https://api.github.com/users/martindurant/gists{/gist_id}","starred_url":"https://api.github.com/users/martindurant/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/martindurant/subscriptions","organizations_url":"https://api.github.com/users/martindurant/orgs","repos_url":"https://api.github.com/users/martindurant/repos","events_url":"https://api.github.com/users/martindurant/events{/privacy}","received_events_url":"https://api.github.com/users/martindurant/received_events","type":"User","site_admin":false},"body":"Should there be a pending state before at least one chunk of a given file has started?","created_at":"2016-09-14T17:12:48Z","updated_at":"2016-09-14T17:12:48Z","html_url":"https://github.com/Azure/azure-data-lake-store-python/pull/64#discussion_r78792200","pull_request_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/64","_links":{"self":{"href":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/comments/78792200"},"html":{"href":"https://github.com/Azure/azure-data-lake-store-python/pull/64#discussion_r78792200"},"pull_request":{"href":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/64"}}},"pull_request":{"url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/64","id":85134562,"html_url":"https://github.com/Azure/azure-data-lake-store-python/pull/64","diff_url":"https://github.com/Azure/azure-data-lake-store-python/pull/64.diff","patch_url":"https://github.com/Azure/azure-data-lake-store-python/pull/64.patch","issue_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/issues/64","number":64,"state":"open","locked":false,"title":"PR: Downloader/uploader improvements","user":{"login":"jbcrail","id":6038,"avatar_url":"https://avatars.githubusercontent.com/u/6038?v=3","gravatar_id":"","url":"https://api.github.com/users/jbcrail","html_url":"https://github.com/jbcrail","followers_url":"https://api.github.com/users/jbcrail/followers","following_url":"https://api.github.com/users/jbcrail/following{/other_user}","gists_url":"https://api.github.com/users/jbcrail/gists{/gist_id}","starred_url":"https://api.github.com/users/jbcrail/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbcrail/subscriptions","organizations_url":"https://api.github.com/users/jbcrail/orgs","repos_url":"https://api.github.com/users/jbcrail/repos","events_url":"https://api.github.com/users/jbcrail/events{/privacy}","received_events_url":"https://api.github.com/users/jbcrail/received_events","type":"User","site_admin":false},"body":"This PR will give us improved control of transfer activity and will allow us to save/resume/monitor with more detail. The common transfer functionality has been refactored into a separate module.\r\n\r\nWhether uploading or downloading, files can be written into temporary files and merged at the end. Previously, this was only available when uploading.\r\n\r\nThe actual callables used to transfer and merge can be overridden, making it easier to add unit tests. For example, the transfer/merge callables can be no-ops, thus, testing core functionality without network traffic.\r\n \r\nOther miscellaneous improvements:\r\n* Add fine-grained states for file and chunk statuses\r\n* Add hooks to view transfer progress (helps with #39)\r\n* Add parameters to tune temporary files\r\n* Add timeout to transfer monitor\r\n* Shutdown transfer more cleanly\r\n\r\n~~Note: persistence is currently broken and will be fixed soon.~~","created_at":"2016-09-13T22:08:16Z","updated_at":"2016-09-14T17:12:48Z","closed_at":null,"merged_at":null,"merge_commit_sha":"3b80534383410e427ca0363eca57af08c05dd498","assignee":{"login":"martindurant","id":6042212,"avatar_url":"https://avatars.githubusercontent.com/u/6042212?v=3","gravatar_id":"","url":"https://api.github.com/users/martindurant","html_url":"https://github.com/martindurant","followers_url":"https://api.github.com/users/martindurant/followers","following_url":"https://api.github.com/users/martindurant/following{/other_user}","gists_url":"https://api.github.com/users/martindurant/gists{/gist_id}","starred_url":"https://api.github.com/users/martindurant/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/martindurant/subscriptions","organizations_url":"https://api.github.com/users/martindurant/orgs","repos_url":"https://api.github.com/users/martindurant/repos","events_url":"https://api.github.com/users/martindurant/events{/privacy}","received_events_url":"https://api.github.com/users/martindurant/received_events","type":"User","site_admin":false},"assignees":[{"login":"martindurant","id":6042212,"avatar_url":"https://avatars.githubusercontent.com/u/6042212?v=3","gravatar_id":"","url":"https://api.github.com/users/martindurant","html_url":"https://github.com/martindurant","followers_url":"https://api.github.com/users/martindurant/followers","following_url":"https://api.github.com/users/martindurant/following{/other_user}","gists_url":"https://api.github.com/users/martindurant/gists{/gist_id}","starred_url":"https://api.github.com/users/martindurant/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/martindurant/subscriptions","organizations_url":"https://api.github.com/users/martindurant/orgs","repos_url":"https://api.github.com/users/martindurant/repos","events_url":"https://api.github.com/users/martindurant/events{/privacy}","received_events_url":"https://api.github.com/users/martindurant/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/64/commits","review_comments_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/64/comments","review_comment_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/comments{/number}","comments_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/issues/64/comments","statuses_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/statuses/dc06662b7c3099414d37f0667ef121d26803198d","head":{"label":"jbcrail:refactor-transfer","ref":"refactor-transfer","sha":"dc06662b7c3099414d37f0667ef121d26803198d","user":{"login":"jbcrail","id":6038,"avatar_url":"https://avatars.githubusercontent.com/u/6038?v=3","gravatar_id":"","url":"https://api.github.com/users/jbcrail","html_url":"https://github.com/jbcrail","followers_url":"https://api.github.com/users/jbcrail/followers","following_url":"https://api.github.com/users/jbcrail/following{/other_user}","gists_url":"https://api.github.com/users/jbcrail/gists{/gist_id}","starred_url":"https://api.github.com/users/jbcrail/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbcrail/subscriptions","organizations_url":"https://api.github.com/users/jbcrail/orgs","repos_url":"https://api.github.com/users/jbcrail/repos","events_url":"https://api.github.com/users/jbcrail/events{/privacy}","received_events_url":"https://api.github.com/users/jbcrail/received_events","type":"User","site_admin":false},"repo":{"id":65852329,"name":"azure-data-lake-store-python","full_name":"jbcrail/azure-data-lake-store-python","owner":{"login":"jbcrail","id":6038,"avatar_url":"https://avatars.githubusercontent.com/u/6038?v=3","gravatar_id":"","url":"https://api.github.com/users/jbcrail","html_url":"https://github.com/jbcrail","followers_url":"https://api.github.com/users/jbcrail/followers","following_url":"https://api.github.com/users/jbcrail/following{/other_user}","gists_url":"https://api.github.com/users/jbcrail/gists{/gist_id}","starred_url":"https://api.github.com/users/jbcrail/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbcrail/subscriptions","organizations_url":"https://api.github.com/users/jbcrail/orgs","repos_url":"https://api.github.com/users/jbcrail/repos","events_url":"https://api.github.com/users/jbcrail/events{/privacy}","received_events_url":"https://api.github.com/users/jbcrail/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/jbcrail/azure-data-lake-store-python","description":"Microsoft Azure Data Lake Store Filesystem Library for Python","fork":true,"url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python","forks_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/forks","keys_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/teams","hooks_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/hooks","issue_events_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/issues/events{/number}","events_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/events","assignees_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/assignees{/user}","branches_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/branches{/branch}","tags_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/tags","blobs_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/git/refs{/sha}","trees_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/statuses/{sha}","languages_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/languages","stargazers_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/stargazers","contributors_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/contributors","subscribers_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/subscribers","subscription_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/subscription","commits_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/commits{/sha}","git_commits_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/git/commits{/sha}","comments_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/comments{/number}","issue_comment_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/issues/comments{/number}","contents_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/contents/{+path}","compare_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/merges","archive_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/downloads","issues_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/issues{/number}","pulls_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/pulls{/number}","milestones_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/milestones{/number}","notifications_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/labels{/name}","releases_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/releases{/id}","deployments_url":"https://api.github.com/repos/jbcrail/azure-data-lake-store-python/deployments","created_at":"2016-08-16T20:43:34Z","updated_at":"2016-08-16T20:05:18Z","pushed_at":"2016-09-14T06:38:16Z","git_url":"git://github.com/jbcrail/azure-data-lake-store-python.git","ssh_url":"git@github.com:jbcrail/azure-data-lake-store-python.git","clone_url":"https://github.com/jbcrail/azure-data-lake-store-python.git","svn_url":"https://github.com/jbcrail/azure-data-lake-store-python","homepage":null,"size":753,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"Azure:dev","ref":"dev","sha":"50cdb8c89599fffc8bf221ec0b671ac09d39fe32","user":{"login":"Azure","id":6844498,"avatar_url":"https://avatars.githubusercontent.com/u/6844498?v=3","gravatar_id":"","url":"https://api.github.com/users/Azure","html_url":"https://github.com/Azure","followers_url":"https://api.github.com/users/Azure/followers","following_url":"https://api.github.com/users/Azure/following{/other_user}","gists_url":"https://api.github.com/users/Azure/gists{/gist_id}","starred_url":"https://api.github.com/users/Azure/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Azure/subscriptions","organizations_url":"https://api.github.com/users/Azure/orgs","repos_url":"https://api.github.com/users/Azure/repos","events_url":"https://api.github.com/users/Azure/events{/privacy}","received_events_url":"https://api.github.com/users/Azure/received_events","type":"Organization","site_admin":false},"repo":{"id":62902455,"name":"azure-data-lake-store-python","full_name":"Azure/azure-data-lake-store-python","owner":{"login":"Azure","id":6844498,"avatar_url":"https://avatars.githubusercontent.com/u/6844498?v=3","gravatar_id":"","url":"https://api.github.com/users/Azure","html_url":"https://github.com/Azure","followers_url":"https://api.github.com/users/Azure/followers","following_url":"https://api.github.com/users/Azure/following{/other_user}","gists_url":"https://api.github.com/users/Azure/gists{/gist_id}","starred_url":"https://api.github.com/users/Azure/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Azure/subscriptions","organizations_url":"https://api.github.com/users/Azure/orgs","repos_url":"https://api.github.com/users/Azure/repos","events_url":"https://api.github.com/users/Azure/events{/privacy}","received_events_url":"https://api.github.com/users/Azure/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Azure/azure-data-lake-store-python","description":"Microsoft Azure Data Lake Store Filesystem Library for Python","fork":false,"url":"https://api.github.com/repos/Azure/azure-data-lake-store-python","forks_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/forks","keys_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/teams","hooks_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/hooks","issue_events_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/issues/events{/number}","events_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/events","assignees_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/assignees{/user}","branches_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/branches{/branch}","tags_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/tags","blobs_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/git/refs{/sha}","trees_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/statuses/{sha}","languages_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/languages","stargazers_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/stargazers","contributors_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/contributors","subscribers_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/subscribers","subscription_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/subscription","commits_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/commits{/sha}","git_commits_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/git/commits{/sha}","comments_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/comments{/number}","issue_comment_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/issues/comments{/number}","contents_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/contents/{+path}","compare_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/merges","archive_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/downloads","issues_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/issues{/number}","pulls_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls{/number}","milestones_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/milestones{/number}","notifications_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/labels{/name}","releases_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/releases{/id}","deployments_url":"https://api.github.com/repos/Azure/azure-data-lake-store-python/deployments","created_at":"2016-07-08T16:31:32Z","updated_at":"2016-08-16T20:05:18Z","pushed_at":"2016-09-14T06:38:16Z","git_url":"git://github.com/Azure/azure-data-lake-store-python.git","ssh_url":"git@github.com:Azure/azure-data-lake-store-python.git","clone_url":"https://github.com/Azure/azure-data-lake-store-python.git","svn_url":"https://github.com/Azure/azure-data-lake-store-python","homepage":null,"size":1341,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":20,"forks":1,"open_issues":20,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/64"},"html":{"href":"https://github.com/Azure/azure-data-lake-store-python/pull/64"},"issue":{"href":"https://api.github.com/repos/Azure/azure-data-lake-store-python/issues/64"},"comments":{"href":"https://api.github.com/repos/Azure/azure-data-lake-store-python/issues/64/comments"},"review_comments":{"href":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/64/comments"},"review_comment":{"href":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/Azure/azure-data-lake-store-python/pulls/64/commits"},"statuses":{"href":"https://api.github.com/repos/Azure/azure-data-lake-store-python/statuses/dc06662b7c3099414d37f0667ef121d26803198d"}}}}
{"actor":{"display_login":"martindurant"}}