{"action":"created","comment":{"url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/pulls/comments/165881362","pull_request_review_id":93881142,"id":165881362,"diff_hunk":"@@ -0,0 +1,1109 @@\n+import json\n+import uuid\n+import base64\n+import typing\n+import hashlib\n+from http import HTTPStatus\n+\n+from waterbutler.core.path import WaterButlerPath\n+from waterbutler.core.provider import BaseProvider\n+from waterbutler.core.streams import BaseStream, HashStreamWriter, ResponseStreamReader\n+from waterbutler.core.metadata import BaseMetadata, BaseFileMetadata\n+from waterbutler.core.exceptions import (MetadataError, NotFoundError, CreateFolderError,\n+                                         UploadError, UploadChecksumMismatchError,\n+                                         DownloadError, DeleteError, CopyError, WaterButlerError)\n+\n+from waterbutler.providers.googlecloud import utils as pd_utils\n+from waterbutler.providers.googlecloud import settings as pd_settings\n+from waterbutler.providers.googlecloud.metadata import (BaseGoogleCloudMetadata,\n+                                                        GoogleCloudFileMetadata,\n+                                                        GoogleCloudFolderMetadata,)\n+\n+\n+class GoogleCloudProvider(BaseProvider):\n+    \"\"\"Provider for Google's Cloud Storage Service.\n+\n+    General API Docs: https://cloud.google.com/storage/docs/apis\n+    JSON API Docs: https://cloud.google.com/storage/docs/json_api/v1/\n+\n+    Please note the name of the service is \"Cloud Storage\" on \"Google Cloud Platform\".  However,\n+    it is named ``GoogleCloudProvider`` for better clarity and consistency.  \"Google Cloud\", \"Cloud\n+    Storage\" and \"Google Cloud Storage\" are used interchangeably.\n+\n+    TODO: should we add a brief \"quirks\" list/abstract here? Details have been provided in the code.\n+    \"\"\"\n+\n+    NAME = pd_settings.NAME\n+    BASE_URL = pd_settings.BASE_URL\n+    UPLOAD_URL = pd_settings.UPLOAD_URL\n+    BATCH_URL = pd_settings.BATCH_URL\n+\n+    def __init__(self, auth: dict, credentials: dict, settings: dict):\n+        \"\"\"Initialize a provider instance with the given params.\n+\n+        1. Here is an example of the settings for the \"osfstorage\" addon in OSF.\n+\n+            WATERBUTLER_CREDENTIALS = {\n+                'storage': {\n+                    'token': 'change_me',\n+                }\n+            }\n+\n+            WATERBUTLER_SETTINGS = {\n+                'storage': {\n+                    'provider': 'change_me',\n+                    'bucket': 'change_me',\n+                    'region': 'change_me',\n+                },\n+            }\n+\n+            WATERBUTLER_RESOURCE = 'bucket'\n+\n+        TODO: the settings may change, update ``__init__()`` when that happens\n+        TODO: what does ``WATERBUTLER_RESOURCE`` do?\n+\n+        2. More about authentication and authorization for Google Cloud\n+\n+        Although waterbutler does not handle authentication/authorization, it is still worthwhile to\n+        understand how Google Cloud works.  As mentioned below in their documentations, Google Cloud\n+        uses OAuth 2.0.  Waterbutler obtains the access token from OSF and set the \"Authorization\"\n+        header in every API requests.\n+\n+        Docs: https://cloud.google.com/storage/docs/authentication\n+\n+        \"Cloud Storage uses OAuth 2.0 for API authentication and authorization. Authentication is\n+        the process of determining the identity of a client.\"\n+\n+        \"A server-centric flow allows an application to directly hold the credentials of a service\n+        account to complete authentication.\"\n+\n+        \"When you use a service account to authenticate your application, you do not need a user to\n+        authenticate to get an access token. Instead, you obtain a private key from the Google Cloud\n+        Platform Console, which you then use to send a signed request for an access token.\"\n+\n+        TODO: access token by default expires after an hour, OSF side needs to take care of it\n+        \"\"\"\n+\n+        super().__init__(auth, credentials, settings)\n+\n+        self.access_token = credentials.get('token')\n+        self.bucket = settings.get('bucket')\n+        self.region = settings.get('region')\n+\n+    async def metadata(\n+            self,\n+            path: WaterButlerPath,\n+            **kwargs\n+    ) -> typing.Union[BaseGoogleCloudMetadata, typing.List[BaseGoogleCloudMetadata]]:\n+        \"\"\"Get the metadata about the object with the given Waterbutler path.\n+\n+        TODO: Verify the claim below\n+        Note: ``OSFStorageProvider`` never uses the inner provider to call ``metadata`` on folders\n+\n+        :param path: the Waterbutler path to the file or folder\n+        :param kwargs: additional kwargs are ignored\n+        :rtype BaseGoogleCloudMetadata: for file\n+        :rtype List<BaseGoogleCloudMetadata>: for folder\n+        \"\"\"\n+\n+        if path.is_folder:\n+            return await self._metadata_folder(path)\n+        else:\n+            return await self._metadata_object(path, is_folder=False)\n+\n+    async def _metadata_object(\n+            self,\n+            path: WaterButlerPath,\n+            is_folder: bool=False\n+    ) -> BaseGoogleCloudMetadata:\n+        \"\"\"Get the metadata about the object itself with the given Waterbutler path.\n+\n+        Google Cloud Storage: Objects - Get\n+        JSON API Docs: https://cloud.google.com/storage/docs/json_api/v1/objects/get\n+\n+        \"By default, this responds with an object resource in the response body. If you provide the\n+        URL parameter alt=media, then it will respond with the object data in the response body.\"\n+\n+        Please note that ``is_folder`` flag is explicitly used. Providing the wrong type will fail\n+        all the time. This is also true for many internal/private/helper/utility methods of/for this\n+        class. They are not exposed to usage outside, including the parent classes.\n+\n+        :param path: the Waterbutler path of the object\n+        :param is_folder: whether the object is a file or folder\n+        :rtype BaseGoogleCloudMetadata\n+        \"\"\"\n+\n+        http_method = 'GET'\n+        obj_name = pd_utils.get_obj_name(path, is_folder=is_folder)\n+        metadata_url = self.build_url(base_url=pd_settings.BASE_URL, obj_name=obj_name)\n+\n+        resp = await self.make_request(\n+            http_method,\n+            metadata_url,\n+            expects=(HTTPStatus.OK,),\n+            throws=MetadataError\n+        )\n+\n+        try:\n+            data = await resp.json()\n+        except (TypeError, json.JSONDecodeError):\n+            raise MetadataError('Failed to parse response, expecting JSON format.')\n+\n+        if is_folder:\n+            return GoogleCloudFolderMetadata(data)\n+        else:\n+            return GoogleCloudFileMetadata(data)\n+\n+    async def _metadata_folder(\n+            self,\n+            path: WaterButlerPath\n+    ) -> typing.List[BaseGoogleCloudMetadata]:\n+        \"\"\"Get the metadata about the folder's immediate children with the given Waterbutler path.\n+\n+        Google Cloud Storage: Objects - List\n+        JSON API Docs: https://cloud.google.com/storage/docs/json_api/v1/objects/list\n+\n+        \"In conjunction with the prefix filter, the use of the delimiter parameter allows the list\n+        method to operate like a directory listing, despite the object namespace being flat.\"\n+\n+        Set the object name of the folder as the PREFIX, and use '/' as the DELIMITER.  For example:\n+\n+            \"storage/v1/b/gcloud-test.longzechen.com/o?prefix=test-folder-1%2F&delimiter=%2F\".\n+\n+        Below is an example of the structure of the JSON response. \"items\" contains the folder\n+        itself and all its immediate child files.  \"prefixes\" contains all its immediate child\n+        folders.  Refer to the  file \"tests/googlecloud/fixtures/metadata/folder-all.json\" for a\n+        real example.\n+\n+            {\n+                \"kind\": \"storage#objects\",\n+                \"prefixes\": [\n+                    \"test-folder-1/test-folder-5/\",\n+                ],\n+                \"items\": [\n+                    {\n+                        \"name\": \"test-folder-1/\",\n+                    },\n+                    {\n+                        \"name\": \"test-folder-1/DSC_0235.JPG\",\n+                    },\n+                ]\n+            }\n+\n+        :param path: the Waterbutler path of the folder\n+        :rtype List<BaseGoogleCloudMetadata>\n+        \"\"\"\n+\n+        # Retrieve a list of metadata for all immediate children (both files and folders)\n+        http_method = 'GET'\n+        prefix = pd_utils.get_obj_name(path, is_folder=True)\n+        delimiter = '/'\n+        query = {\n+            'prefix': prefix,\n+            'delimiter': delimiter\n+        }\n+        metadata_url = self.build_url(base_url=pd_settings.BASE_URL, **query)\n+\n+        resp = await self.make_request(\n+            http_method,\n+            metadata_url,\n+            expects=(HTTPStatus.OK,),\n+            throws=MetadataError\n+        )\n+\n+        try:\n+            data = await resp.json()\n+        except (TypeError, json.JSONDecodeError):\n+            raise MetadataError('Failed to parse response, expecting JSON format.')\n+\n+        # If folder does not exist, raise HTTP 404 Not Found\n+        prefixes = data.get('prefixes', None)\n+        items = data.get('items', None)\n+        if not items and not prefixes:\n+            raise NotFoundError(path.path)\n+\n+        # Add immediate child files to metadata list, need to exclude itself\n+        folder_metadata_list = []\n+        for item in data.get('items', []):\n+            if item.get('name', '') != prefix:\n+                folder_metadata_list.append(GoogleCloudFileMetadata(item))\n+\n+        # Retrieve and add immediate child folders to the metadata list\n+        sub_folder_names = data.get('prefixes', [])\n+        for name in sub_folder_names:\n+            path = pd_utils.build_path(name, is_folder=True)\n+            sub_folder_metadata = await self._metadata_object(\n+                WaterButlerPath(path),\n+                is_folder=True\n+            )\n+            folder_metadata_list.append(sub_folder_metadata)\n+\n+        return folder_metadata_list\n+\n+    async def _metadata_all_children(\n+            self,\n+            path: WaterButlerPath\n+    ) -> typing.List[dict]:\n+        \"\"\"Get the metadata about all of the folder's children with the given Waterbutler path.\n+\n+        Google Cloud Storage: Objects - List\n+        JSON API Docs: https://cloud.google.com/storage/docs/json_api/v1/objects/list\n+\n+        In Google Cloud Storage, find all children of a given folder is equal to finding all objects\n+        of which the names are prefixed with the folder's object name.  Note that the folder itself\n+        is included in the result.  Set the object name of the folder as the PREFIX, and do not use\n+        a DELIMITER.  For example:\n+\n+            \"/storage/v1/b/gcloud-test.longzechen.com/o?prefix=test-folder-1%2F\"\n+\n+        :param path: the Waterbutler path of the folder\n+        :rtype List<BaseGoogleCloudMetadata>\n+        \"\"\"\n+\n+        http_method = 'GET'\n+        prefix = pd_utils.get_obj_name(path, is_folder=True)\n+        query = {'prefix': prefix}\n+        metadata_url = self.build_url(base_url=pd_settings.BASE_URL, **query)\n+\n+        resp = await self.make_request(\n+            http_method,\n+            metadata_url,\n+            expects=(HTTPStatus.OK,),\n+            throws=MetadataError\n+        )\n+\n+        try:\n+            data = await resp.json()\n+        except (TypeError, json.JSONDecodeError):\n+            raise MetadataError('Failed to parse response, expecting JSON format.')\n+\n+        return data.get('items', [])\n+\n+    async def _exists_folder(self, path: WaterButlerPath) -> bool:\n+        \"\"\"Check if a folder with the given Waterbutler path exists. Calls ``_metadata_object()``.\n+\n+        For folders, ``exists()`` from the core provider calls ``metadata()``, which further calls\n+        ``_metadata_folder``.  This makes simple action more complicated and more expensive.\n+\n+        However, ``exists()`` for files does not have this limitation.\n+\n+        :param path: the Waterbutler path of the folder to check\n+        :rtype bool:\n+        \"\"\"\n+\n+        if not path.is_folder:\n+            raise WaterButlerError('Expecting folder instead of file')\n+\n+        try:\n+            await self._metadata_object(path, is_folder=True)\n+        except NotFoundError:\n+            return False\n+        except MetadataError as exc:\n+            if exc.code != HTTPStatus.NOT_FOUND:\n+                raise\n+            return False\n+        return True\n+\n+    async def create_folder(\n+            self,\n+            path: WaterButlerPath,\n+            **kwargs\n+    ) -> BaseGoogleCloudMetadata:\n+        \"\"\"Create a folder with the given Waterbutler path.\n+\n+        Google Cloud Storage: Objects - Insert\n+        JSON API Docs: https://cloud.google.com/storage/docs/json_api/v1/objects/insert\n+\n+        Create a folder in the Google Cloud is basically to upload an object (which is a folder or\n+        of which the name ends with a `/`). First, the size of the object (folder) is set to 0.\n+        Second, the Content-Type header is set to: application/x-www-form-urlencoded;charset=UTF-8\n+\n+        1. It is possible to create a folder \"/a/b/c/\" when neither \"/a/\" or \"/a/b/\" exists.\n+\n+        2. When creating a folder, if it already exists, throw a HTTP 409 Conflict\n+\n+        3. ``create_folder``, though fully implemented, is not required for the limited version.\n+\n+\n+        TODO: Verify the claim below\n+        Note: ``OSFStorageProvider`` never uses the inner provider to call ``create_folder``\n+\n+        :param path: the Waterbutler path of the folder to create\n+        :param kwargs: additional kwargs are ignored\n+        :rtype: BaseGoogleCloudMetadata\n+        \"\"\"\n+\n+        if await self._exists_folder(path):\n+            raise CreateFolderError('Folder already exists.', code=HTTPStatus.CONFLICT)\n+\n+        http_method = 'POST'\n+        query = {\n+            'uploadType': 'media',\n+            'name': pd_utils.get_obj_name(path, is_folder=True)\n+        }\n+        upload_url = self.build_url(base_url=pd_settings.UPLOAD_URL, **query)\n+        headers = {\n+            'Content-Length': '0',\n+            'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'\n+        }\n+\n+        resp = await self.make_request(\n+            http_method,\n+            upload_url,\n+            headers=headers,\n+            expects=(HTTPStatus.OK,),\n+            throws=CreateFolderError\n+        )\n+\n+        try:\n+            data = await resp.json()\n+        except (TypeError, json.JSONDecodeError):\n+            raise MetadataError('Failed to parse response, expecting JSON format.')\n+\n+        return GoogleCloudFolderMetadata(data)\n+\n+    async def upload(\n+            self,\n+            stream: BaseStream,\n+            path: WaterButlerPath,\n+            *args,\n+            **kwargs\n+    ) -> typing.Tuple[BaseGoogleCloudMetadata, bool]:\n+        \"\"\"Upload a stream with the given Waterbutler path.\n+\n+        Google Cloud Storage: Objects - Insert\n+        JSON API Docs: https://cloud.google.com/storage/docs/json_api/v1/objects/insert\n+\n+        \"In the JSON API, the objects resource md5Hash and crc32c properties contain base64-encoded\n+        MD5 and CRC32c hashes, respectively.\"  Need to encode the MD5 hash calculated by stream\n+        writer.  For more information, refer to https://cloud.google.com/storage/docs/hashes-etags.\n+\n+        :param stream: the stream to post\n+        :param path: the Waterbutler path of the file to upload\n+        :param args: additional args are ignored\n+        :param kwargs: additional kwargs are ignored\n+        :rtype BaseGoogleCloudMetadata:\n+        :rtype bool:\n+        \"\"\"\n+\n+        created = not await self.exists(path)\n+\n+        stream.add_writer('md5', HashStreamWriter(hashlib.md5))\n+\n+        http_method = 'POST'\n+        query = {\n+            'uploadType': 'media',\n+            'name': pd_utils.get_obj_name(path, is_folder=False)\n+        }\n+        upload_url = self.build_url(base_url=pd_settings.UPLOAD_URL, **query)\n+        headers = {'Content-Length': str(stream.size)}\n+\n+        resp = await self.make_request(\n+            http_method,\n+            upload_url,\n+            data=stream,\n+            headers=headers,\n+            expects=(HTTPStatus.OK,),\n+            throws=UploadError\n+        )\n+\n+        try:\n+            data = await resp.json()\n+        except (TypeError, json.JSONDecodeError):\n+            raise MetadataError('Failed to parse response, expecting JSON format.')\n+\n+        # Encode the MD5 hash by stream writer using BASE64\n+        encoded_md5 = base64.b64encode(stream.writers['md5'].digest)\n+        # Encode the MD5 hash from metadata using UTF-8\n+        metadata_md5 = data.get('md5Hash', '').encode('UTF-8')\n+\n+        if encoded_md5 != metadata_md5:\n+            raise UploadChecksumMismatchError\n+\n+        return GoogleCloudFileMetadata(data), created\n+\n+    async def download(\n+            self,\n+            path: WaterButlerPath,\n+            *args,\n+            **kwargs\n+    ) -> ResponseStreamReader:\n+        \"\"\"Download the object with the given path and and return a ``ResponseStreamReader``.\n+\n+        Google Cloud Storage: Objects - Get\n+        JSON API Docs: https://cloud.google.com/storage/docs/json_api/v1/objects/get\n+\n+        \"By default, this responds with an object resource in the response body. If you provide the\n+        URL parameter alt=media, then it will respond with the object data in the response body.\"\n+\n+        Note: download request on folders return HTTP 200 OK with empty body. The action doesn't do\n+        anything. It doesn't make any sense just to download the folder anyway.\n+\n+        TODO: should we support the Range header?\n+        TODO: what is ``accept_url`` and should we support it?\n+\n+        TODO: Verify the claim below\n+        Note: ``OSFStorageProvider`` never uses the inner provider to calls ``download`` on folders\n+\n+        :param path: the Waterbutler path to the object to download\n+        :param args: additional args are ignored\n+        :param kwargs: additional kwargs are ignored\n+        :rtype ResponseStreamReader:\n+        \"\"\"\n+\n+        if path.is_folder:\n+            raise DownloadError('Cannot download folders', code=HTTPStatus.BAD_REQUEST)\n+\n+        http_method = 'GET'\n+        query = {'alt': 'media'}\n+        download_url = self.build_url(\n+            base_url=pd_settings.BASE_URL,\n+            obj_name=pd_utils.get_obj_name(path, is_folder=False),\n+            **query\n+        )\n+        resp = await self.make_request(\n+            http_method,\n+            download_url,\n+            expects=(HTTPStatus.OK,),\n+            throws=DownloadError\n+        )\n+\n+        return ResponseStreamReader(resp)\n+\n+    async def delete(self, path: WaterButlerPath, *args, **kwargs) -> None:\n+        \"\"\"Deletes the object with the specified Waterbutler path.\n+\n+        Google Cloud Storage: Objects - Delete\n+        JSON API Docs: https://cloud.google.com/storage/docs/json_api/v1/objects/delete\n+\n+        \"Deletes an object and its metadata. Deletions are permanent if versioning is not enabled\n+        for the bucket, or if the generation parameter is used.\"\n+\n+        TODO: Verify the claim below\n+        Note: ``OSFStorageProvider`` never uses the inner provider to call ``delete`` on folders\n+\n+        :param path: the Waterbutler path of the object to delete\n+        :param args: additional args are ignored\n+        :param kwargs: additional kwargs are ignored\n+        :rtype None:\n+        \"\"\"\n+\n+        if path.is_folder:\n+            return await self._delete_folder(path)\n+        else:\n+            return await self._delete_file(path)\n+\n+    async def _delete_file(self, path: WaterButlerPath) -> None:\n+        \"\"\"Deletes the file with the specified Waterbutler path.\n+\n+        :param path: the Waterbutler path of the file to delete\n+        :rtype None:\n+        \"\"\"\n+\n+        http_method = 'DELETE'\n+        delete_url = self.build_url(\n+            base_url=pd_settings.BASE_URL,\n+            obj_name=pd_utils.get_obj_name(path)\n+        )\n+\n+        resp = await self.make_request(\n+            http_method,\n+            delete_url,\n+            expects=(HTTPStatus.NO_CONTENT,),\n+            throws=DeleteError,\n+        )\n+\n+        await resp.release()\n+\n+    async def _delete_folder(self, path: WaterButlerPath) -> None:\n+        \"\"\"Deletes the folder and all its content with the specified Waterbutler path.\n+\n+        For Google Cloud Storage, `DELETE` a folder does delete the folder object.  However, given\n+        the storage is flat, everything else in (or more accurately, prefixed with) the folder are\n+        not deleted.  To fully delete a folder and all its contents, we need to delete each of its\n+        children (whether immediate or not). This is both easier and harder that it seems:\n+\n+        1. EASY: No need for recursive behavior.  Every children must have an object name that is\n+           prefixed with the object name of the folder.  And every object that is prefixed with the\n+           object name of the folder must be its children.\n+\n+        2. EASY: Google Cloud Storage provides the option for \"Sending Batch Requests\" and one of\n+           the suggested use case  is \"Deleting many objects\".\n+\n+        3. HARD: However, there are a few limitations with batch request but we HANDLES them.\n+\n+           3.1 \"You're LIMITED to 100 calls in a single batch request.  If you need to make more\n+               calls than that, use multiple batch requests.\"\n+\n+           3.2 \"The server may perform your calls in any order.  Don't count on their being executed\n+               in the order in which you specified them.\" In addition, the request is not atomic.\n+               Some may fail while others succeed.  The response status is HTTP 200 OK even when\n+               individual requests fail.  Need to parse the response to detect and handle failures.\n+\n+        4. Quirk: If the folder does exists, the server returns HTTP 200 OK, not HTTP 404 Not found.","path":"waterbutler/providers/googlecloud/provider.py","position":543,"original_position":543,"commit_id":"606d79e8ac4c39494d1e5eea072ba8c37ea5fdc3","original_commit_id":"606d79e8ac4c39494d1e5eea072ba8c37ea5fdc3","user":{"login":"cslzchen","id":3750414,"avatar_url":"https://avatars2.githubusercontent.com/u/3750414?v=4","gravatar_id":"","url":"https://api.github.com/users/cslzchen","html_url":"https://github.com/cslzchen","followers_url":"https://api.github.com/users/cslzchen/followers","following_url":"https://api.github.com/users/cslzchen/following{/other_user}","gists_url":"https://api.github.com/users/cslzchen/gists{/gist_id}","starred_url":"https://api.github.com/users/cslzchen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cslzchen/subscriptions","organizations_url":"https://api.github.com/users/cslzchen/orgs","repos_url":"https://api.github.com/users/cslzchen/repos","events_url":"https://api.github.com/users/cslzchen/events{/privacy}","received_events_url":"https://api.github.com/users/cslzchen/received_events","type":"User","site_admin":false},"body":"Yep, typo here.","created_at":"2018-02-05T05:00:27Z","updated_at":"2018-02-05T05:20:08Z","html_url":"https://github.com/CenterForOpenScience/waterbutler/pull/317#discussion_r165881362","pull_request_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/pulls/317","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/CenterForOpenScience/waterbutler/pulls/comments/165881362"},"html":{"href":"https://github.com/CenterForOpenScience/waterbutler/pull/317#discussion_r165881362"},"pull_request":{"href":"https://api.github.com/repos/CenterForOpenScience/waterbutler/pulls/317"}},"in_reply_to_id":165713052},"pull_request":{"url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/pulls/317","id":165585392,"html_url":"https://github.com/CenterForOpenScience/waterbutler/pull/317","diff_url":"https://github.com/CenterForOpenScience/waterbutler/pull/317.diff","patch_url":"https://github.com/CenterForOpenScience/waterbutler/pull/317.patch","issue_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/issues/317","number":317,"state":"open","locked":false,"title":"[SVCS-617] Add GoogleCloud Provider Limited to Waterbutler","user":{"login":"cslzchen","id":3750414,"avatar_url":"https://avatars2.githubusercontent.com/u/3750414?v=4","gravatar_id":"","url":"https://api.github.com/users/cslzchen","html_url":"https://github.com/cslzchen","followers_url":"https://api.github.com/users/cslzchen/followers","following_url":"https://api.github.com/users/cslzchen/following{/other_user}","gists_url":"https://api.github.com/users/cslzchen/gists{/gist_id}","starred_url":"https://api.github.com/users/cslzchen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cslzchen/subscriptions","organizations_url":"https://api.github.com/users/cslzchen/orgs","repos_url":"https://api.github.com/users/cslzchen/repos","events_url":"https://api.github.com/users/cslzchen/events{/privacy}","received_events_url":"https://api.github.com/users/cslzchen/received_events","type":"User","site_admin":false},"body":"## Ticket\r\n\r\nhttps://openscience.atlassian.net/browse/SVCS-617\r\n\r\n## Purpose\r\n\r\nAdd a (not so) limited \"GoogleCloud\" provider to Waterbutler to replace Rackspace CloudFiles as the storage backend for OSFStorage.\r\n\r\nJSON API Docs: https://cloud.google.com/storage/docs/json_api/v1/\r\n\r\n\r\nPlease note that the provider implemented in this PR is more than **limited**. It is almost a **full** version except missing the revision part.\r\n\r\n### OSF\r\n\r\nThere will be an OSF side PR or server configuration for:\r\n\r\n- [ ] Enable `GoogleCloud` as the storage backend for OSFStorage\r\n- [ ] Obtain (and refresh if necessary) access token using signed request\r\n\r\n### Important Quirks for Google Cloud:\r\n\r\nPlease refer to the docstr for each class/method for detailed information.\r\n\r\n* **Object Name** and slash `/` encoding\r\n* File and folder both as objects\r\n* Non-atomic batch requests\r\n\r\n### Interesting Quirks for OSF Storage\r\n\r\nDuring the integration test, I also went through how the **outer provider** `OSFStorageProvider` uses its **inner provider** `GoogleCloud` (this PR) and `CloudFiles` (before). The outer only makes or calls the inner during `.upload()` and `.download()`. Here are the inner provider's methods that gets called:\r\n\r\n* `.metadata()` on file\r\n* `.upload()` on file\r\n* `.intra_copy()` on file\r\n* `.delete()` on file\r\n* `.download()` on file\r\n\r\n## Changes\r\n\r\n### Utils\r\n\r\n* Customized ``build_url()`` that encode slashes `/` in path and query params.\r\n* Name and path helpers that handles the conversion between **Object Name**, `String` path and `WaterButlerPath`.\r\n* Response parsers for batch delete and copy request.\r\n\r\n### Metadata\r\n\r\n* Class `BaseGoogleCloudMetadata`: An **abstract** class that inherits `BaseMetadata`\r\n* Class `GoogleCloudFileMetadata`: the class for file objects\r\n* Class `GoogleCloudFolderMetadata`: the class for folder objects\r\n\r\nNote: Other providers use extra metadata classes and/or extra inheritance. However, there is no need for `GoogleCloudProvider`.\r\n\r\n### Provider\r\n\r\n* Validate path\r\n* Retrieve metadata for file and folders\r\n* Check existence\r\n* Upload file and create folder\r\n* Download file\r\n* Delete file and folder\r\n* Intra copy file and folder\r\n\r\n### Tests\r\n\r\nThe (unit) testing focuses on three parts. The first part is on building request, both URL and payload, and on parsing response. The request and response in the fixtures are real ones with slight modification. The second part is on **endpoint** methods that are used outside the class. Request are registered using `aiohttpretty` and real response are used to mock the response. The last one is testing that metadata are initialized correctly.\r\n\r\n## Side effects\r\n\r\nTBD\r\n\r\n## QA Notes\r\n\r\nFull QA required, everything you can test on OSF Storage.\r\n\r\n## Deployment Notes\r\n\r\nTBD\r\n\r\n## Extras\r\n\r\nGoogle Cloud is successfully integrated as the OSF Storage backend locally. Here is a few example screen shots.\r\n\r\n![gcloud-files](https://user-images.githubusercontent.com/3750414/35632927-0b9ea55e-0676-11e8-9146-cbae3774a498.png)\r\n\r\n<img width=\"641\" alt=\"osf-files\" src=\"https://user-images.githubusercontent.com/3750414/35632945-134bba44-0676-11e8-93be-e123e4d6cd50.png\">\r\n\r\n","created_at":"2018-01-28T20:23:51Z","updated_at":"2018-02-05T05:20:09Z","closed_at":null,"merged_at":null,"merge_commit_sha":"ed865e8ca1e4858c886dfec2133df1544a197a13","assignee":null,"assignees":[],"requested_reviewers":[{"login":"icereval","id":3111541,"avatar_url":"https://avatars0.githubusercontent.com/u/3111541?v=4","gravatar_id":"","url":"https://api.github.com/users/icereval","html_url":"https://github.com/icereval","followers_url":"https://api.github.com/users/icereval/followers","following_url":"https://api.github.com/users/icereval/following{/other_user}","gists_url":"https://api.github.com/users/icereval/gists{/gist_id}","starred_url":"https://api.github.com/users/icereval/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/icereval/subscriptions","organizations_url":"https://api.github.com/users/icereval/orgs","repos_url":"https://api.github.com/users/icereval/repos","events_url":"https://api.github.com/users/icereval/events{/privacy}","received_events_url":"https://api.github.com/users/icereval/received_events","type":"User","site_admin":false}],"requested_teams":[],"milestone":null,"commits_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/pulls/317/commits","review_comments_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/pulls/317/comments","review_comment_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/pulls/comments{/number}","comments_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/issues/317/comments","statuses_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/statuses/606d79e8ac4c39494d1e5eea072ba8c37ea5fdc3","head":{"label":"cslzchen:longze/feature/svcs-617-googlecloud","ref":"longze/feature/svcs-617-googlecloud","sha":"606d79e8ac4c39494d1e5eea072ba8c37ea5fdc3","user":{"login":"cslzchen","id":3750414,"avatar_url":"https://avatars2.githubusercontent.com/u/3750414?v=4","gravatar_id":"","url":"https://api.github.com/users/cslzchen","html_url":"https://github.com/cslzchen","followers_url":"https://api.github.com/users/cslzchen/followers","following_url":"https://api.github.com/users/cslzchen/following{/other_user}","gists_url":"https://api.github.com/users/cslzchen/gists{/gist_id}","starred_url":"https://api.github.com/users/cslzchen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cslzchen/subscriptions","organizations_url":"https://api.github.com/users/cslzchen/orgs","repos_url":"https://api.github.com/users/cslzchen/repos","events_url":"https://api.github.com/users/cslzchen/events{/privacy}","received_events_url":"https://api.github.com/users/cslzchen/received_events","type":"User","site_admin":false},"repo":{"id":46802949,"name":"waterbutler","full_name":"cslzchen/waterbutler","owner":{"login":"cslzchen","id":3750414,"avatar_url":"https://avatars2.githubusercontent.com/u/3750414?v=4","gravatar_id":"","url":"https://api.github.com/users/cslzchen","html_url":"https://github.com/cslzchen","followers_url":"https://api.github.com/users/cslzchen/followers","following_url":"https://api.github.com/users/cslzchen/following{/other_user}","gists_url":"https://api.github.com/users/cslzchen/gists{/gist_id}","starred_url":"https://api.github.com/users/cslzchen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cslzchen/subscriptions","organizations_url":"https://api.github.com/users/cslzchen/orgs","repos_url":"https://api.github.com/users/cslzchen/repos","events_url":"https://api.github.com/users/cslzchen/events{/privacy}","received_events_url":"https://api.github.com/users/cslzchen/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/cslzchen/waterbutler","description":"WaterButler is a Python web application for interacting with various file storage services via a single RESTful API, developed at The Center for Open Science.","fork":true,"url":"https://api.github.com/repos/cslzchen/waterbutler","forks_url":"https://api.github.com/repos/cslzchen/waterbutler/forks","keys_url":"https://api.github.com/repos/cslzchen/waterbutler/keys{/key_id}","collaborators_url":"https://api.github.com/repos/cslzchen/waterbutler/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/cslzchen/waterbutler/teams","hooks_url":"https://api.github.com/repos/cslzchen/waterbutler/hooks","issue_events_url":"https://api.github.com/repos/cslzchen/waterbutler/issues/events{/number}","events_url":"https://api.github.com/repos/cslzchen/waterbutler/events","assignees_url":"https://api.github.com/repos/cslzchen/waterbutler/assignees{/user}","branches_url":"https://api.github.com/repos/cslzchen/waterbutler/branches{/branch}","tags_url":"https://api.github.com/repos/cslzchen/waterbutler/tags","blobs_url":"https://api.github.com/repos/cslzchen/waterbutler/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/cslzchen/waterbutler/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/cslzchen/waterbutler/git/refs{/sha}","trees_url":"https://api.github.com/repos/cslzchen/waterbutler/git/trees{/sha}","statuses_url":"https://api.github.com/repos/cslzchen/waterbutler/statuses/{sha}","languages_url":"https://api.github.com/repos/cslzchen/waterbutler/languages","stargazers_url":"https://api.github.com/repos/cslzchen/waterbutler/stargazers","contributors_url":"https://api.github.com/repos/cslzchen/waterbutler/contributors","subscribers_url":"https://api.github.com/repos/cslzchen/waterbutler/subscribers","subscription_url":"https://api.github.com/repos/cslzchen/waterbutler/subscription","commits_url":"https://api.github.com/repos/cslzchen/waterbutler/commits{/sha}","git_commits_url":"https://api.github.com/repos/cslzchen/waterbutler/git/commits{/sha}","comments_url":"https://api.github.com/repos/cslzchen/waterbutler/comments{/number}","issue_comment_url":"https://api.github.com/repos/cslzchen/waterbutler/issues/comments{/number}","contents_url":"https://api.github.com/repos/cslzchen/waterbutler/contents/{+path}","compare_url":"https://api.github.com/repos/cslzchen/waterbutler/compare/{base}...{head}","merges_url":"https://api.github.com/repos/cslzchen/waterbutler/merges","archive_url":"https://api.github.com/repos/cslzchen/waterbutler/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/cslzchen/waterbutler/downloads","issues_url":"https://api.github.com/repos/cslzchen/waterbutler/issues{/number}","pulls_url":"https://api.github.com/repos/cslzchen/waterbutler/pulls{/number}","milestones_url":"https://api.github.com/repos/cslzchen/waterbutler/milestones{/number}","notifications_url":"https://api.github.com/repos/cslzchen/waterbutler/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/cslzchen/waterbutler/labels{/name}","releases_url":"https://api.github.com/repos/cslzchen/waterbutler/releases{/id}","deployments_url":"https://api.github.com/repos/cslzchen/waterbutler/deployments","created_at":"2015-11-24T16:09:15Z","updated_at":"2017-08-26T06:12:05Z","pushed_at":"2018-01-31T16:09:34Z","git_url":"git://github.com/cslzchen/waterbutler.git","ssh_url":"git@github.com:cslzchen/waterbutler.git","clone_url":"https://github.com/cslzchen/waterbutler.git","svn_url":"https://github.com/cslzchen/waterbutler","homepage":"http://waterbutler.io","size":26508,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":null,"forks":0,"open_issues":0,"watchers":1,"default_branch":"develop"}},"base":{"label":"CenterForOpenScience:develop","ref":"develop","sha":"9c04bf9871d90cfa89e89ed76be7f3a32552760c","user":{"login":"CenterForOpenScience","id":3344584,"avatar_url":"https://avatars1.githubusercontent.com/u/3344584?v=4","gravatar_id":"","url":"https://api.github.com/users/CenterForOpenScience","html_url":"https://github.com/CenterForOpenScience","followers_url":"https://api.github.com/users/CenterForOpenScience/followers","following_url":"https://api.github.com/users/CenterForOpenScience/following{/other_user}","gists_url":"https://api.github.com/users/CenterForOpenScience/gists{/gist_id}","starred_url":"https://api.github.com/users/CenterForOpenScience/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CenterForOpenScience/subscriptions","organizations_url":"https://api.github.com/users/CenterForOpenScience/orgs","repos_url":"https://api.github.com/users/CenterForOpenScience/repos","events_url":"https://api.github.com/users/CenterForOpenScience/events{/privacy}","received_events_url":"https://api.github.com/users/CenterForOpenScience/received_events","type":"Organization","site_admin":false},"repo":{"id":27130042,"name":"waterbutler","full_name":"CenterForOpenScience/waterbutler","owner":{"login":"CenterForOpenScience","id":3344584,"avatar_url":"https://avatars1.githubusercontent.com/u/3344584?v=4","gravatar_id":"","url":"https://api.github.com/users/CenterForOpenScience","html_url":"https://github.com/CenterForOpenScience","followers_url":"https://api.github.com/users/CenterForOpenScience/followers","following_url":"https://api.github.com/users/CenterForOpenScience/following{/other_user}","gists_url":"https://api.github.com/users/CenterForOpenScience/gists{/gist_id}","starred_url":"https://api.github.com/users/CenterForOpenScience/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CenterForOpenScience/subscriptions","organizations_url":"https://api.github.com/users/CenterForOpenScience/orgs","repos_url":"https://api.github.com/users/CenterForOpenScience/repos","events_url":"https://api.github.com/users/CenterForOpenScience/events{/privacy}","received_events_url":"https://api.github.com/users/CenterForOpenScience/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/CenterForOpenScience/waterbutler","description":"WaterButler is a Python web application for interacting with various file storage services via a single RESTful API, developed at Center for Open Science.","fork":false,"url":"https://api.github.com/repos/CenterForOpenScience/waterbutler","forks_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/forks","keys_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/keys{/key_id}","collaborators_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/teams","hooks_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/hooks","issue_events_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/issues/events{/number}","events_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/events","assignees_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/assignees{/user}","branches_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/branches{/branch}","tags_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/tags","blobs_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/git/refs{/sha}","trees_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/git/trees{/sha}","statuses_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/statuses/{sha}","languages_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/languages","stargazers_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/stargazers","contributors_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/contributors","subscribers_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/subscribers","subscription_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/subscription","commits_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/commits{/sha}","git_commits_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/git/commits{/sha}","comments_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/comments{/number}","issue_comment_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/issues/comments{/number}","contents_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/contents/{+path}","compare_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/compare/{base}...{head}","merges_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/merges","archive_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/downloads","issues_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/issues{/number}","pulls_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/pulls{/number}","milestones_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/milestones{/number}","notifications_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/labels{/name}","releases_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/releases{/id}","deployments_url":"https://api.github.com/repos/CenterForOpenScience/waterbutler/deployments","created_at":"2014-11-25T14:42:31Z","updated_at":"2017-12-03T04:24:11Z","pushed_at":"2018-01-31T16:09:35Z","git_url":"git://github.com/CenterForOpenScience/waterbutler.git","ssh_url":"git@github.com:CenterForOpenScience/waterbutler.git","clone_url":"https://github.com/CenterForOpenScience/waterbutler.git","svn_url":"https://github.com/CenterForOpenScience/waterbutler","homepage":"","size":3333,"stargazers_count":38,"watchers_count":38,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":58,"mirror_url":null,"archived":false,"open_issues_count":37,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":58,"open_issues":37,"watchers":38,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/CenterForOpenScience/waterbutler/pulls/317"},"html":{"href":"https://github.com/CenterForOpenScience/waterbutler/pull/317"},"issue":{"href":"https://api.github.com/repos/CenterForOpenScience/waterbutler/issues/317"},"comments":{"href":"https://api.github.com/repos/CenterForOpenScience/waterbutler/issues/317/comments"},"review_comments":{"href":"https://api.github.com/repos/CenterForOpenScience/waterbutler/pulls/317/comments"},"review_comment":{"href":"https://api.github.com/repos/CenterForOpenScience/waterbutler/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/CenterForOpenScience/waterbutler/pulls/317/commits"},"statuses":{"href":"https://api.github.com/repos/CenterForOpenScience/waterbutler/statuses/606d79e8ac4c39494d1e5eea072ba8c37ea5fdc3"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"cslzchen"}}