{"action":"created","comment":{"url":"https://api.github.com/repos/s3ql/s3ql/pulls/comments/51655275","id":51655275,"diff_hunk":"@@ -0,0 +1,345 @@\n+'''\n+gdrive.py - this file is part of S3QL.\n+\n+Copyright © 2016 Max Khon <fjoe@samodelkin.net>\n+\n+This work can be distributed under the terms of the GNU GPLv3.\n+'''\n+\n+from ..logging import logging # Ensure use of custom logger class\n+from ..inherit_docstrings import (copy_ancestor_docstring, ABCDocstMeta)\n+from .common import (AbstractBackend, DanglingStorageURLError, NoSuchObject)\n+import apiclient\n+import base64\n+import httplib2\n+import io\n+import json\n+import oauth2client.client\n+\n+log = logging.getLogger(__name__)\n+\n+class Backend(AbstractBackend, metaclass=ABCDocstMeta):\n+    '''\n+    A backend that stores data in Google Drive.\n+    '''\n+\n+    known_options = set()\n+\n+    MIME_TYPE_BINARY = 'application/octet-stream'\n+    MIME_TYPE_FOLDER = 'application/vnd.google-apps.folder'\n+\n+    MAX_PROPERTY_SIZE = 124\n+    MAX_REF_CHUNK_SIZE = 100\n+    VALUE_BASE64 = 'b64:'\n+    VALUE_REF = 'ref:'\n+\n+    def __init__(self, storage_url, login, password, options):\n+        '''Initialize local backend\n+\n+        Login and password are ignored.\n+        '''\n+        # Unused argument\n+        #pylint: disable=W0613\n+\n+        super().__init__()\n+        self.login = login\n+        self.password = password\n+        self.options = options\n+\n+        # get google drive service\n+        credentials = self._get_credentials()\n+        http = credentials.authorize(httplib2.Http())\n+        self.service = apiclient.discovery.build('drive', 'v3', http=http)\n+\n+        # get gdrive folder\n+        path = storage_url[len('gdrive://'):].rstrip('/')\n+        self.folder = self._lookup_file(path)\n+        if self.folder is None or self.folder['mimeType'] != Backend.MIME_TYPE_FOLDER:\n+            raise DanglingStorageURLError(self.path)\n+\n+    def _get_credentials(self):\n+        credentials_filename = oauth2client.client._get_environment_variable_file()\n+        if not credentials_filename:\n+            credentials_filename = oauth2client.client._get_well_known_file()\n+        if not credentials_filename:\n+            return None\n+        with open(credentials_filename) as f:\n+            creds = json.load(f)\n+\n+        credentials = oauth2client.client.SignedJwtAssertionCredentials(\n+            creds['client_email'],\n+            creds['private_key'],\n+            'https://www.googleapis.com/auth/drive',\n+            sub=self.login)\n+        return credentials\n+\n+    def _list_files(self, folder, fields):\n+        # list folder children\n+        page_token = None\n+        while True:\n+            results = self.service.files().list(\n+                q=\"'{0}' in parents\".format(folder['id']),\n+                pageToken=page_token,\n+                fields=\"nextPageToken, files(%s)\" % fields).execute()\n+            for f in results.get('files', []):\n+                yield f\n+\n+            # get next page\n+            page_token = results.get('nextPageToken', None)\n+            if page_token is None:\n+                break\n+\n+    def _lookup_file(self, name, folder=None):\n+        if folder is None:\n+            folder = { 'id': 'root', 'path': '/', 'mimeType': Backend.MIME_TYPE_FOLDER }\n+\n+        # find next path component\n+        if len(name) == 0:\n+            return folder\n+        if name[0] == '/':\n+            name = name[1:]\n+        n = name.split('/', 1)\n+        if len(n[0]) == 0:\n+            return folder\n+\n+        # iterate over folder children\n+        for f in self._list_files(folder, \"id, name, mimeType, size, properties\"):\n+            f['path'] = folder['path'] + f['name']\n+            if f['mimeType'] == Backend.MIME_TYPE_FOLDER:\n+                f['path'] += '/'\n+\n+            #log.debug(\"{0} ({1}: {2})\".format(f['path'], f['id'], f['mimeType']))\n+            if f['name'] != n[0]:\n+                continue\n+\n+            # found matching child\n+            if len(n) == 1:\n+                return f\n+            elif f['mimeType'] == Backend.MIME_TYPE_FOLDER:\n+                return self._lookup_file(n[1], f)\n+            else:\n+                return None\n+\n+        # nothing found\n+        return None\n+\n+    def _create_file(self, body):\n+        return self.service.files().create(body=body).execute()\n+\n+    def _copy_file(self, fileId, body):\n+        return self.service.files().copy(fileId=fileId, body=body).execute()\n+\n+    def _delete_file(self, f):\n+        self.service.files().delete(fileId=f['id']).execute()\n+\n+    def _delete_other(self, f):","path":"src/s3ql/backends/gdrive.py","position":135,"original_position":135,"commit_id":"4fe6a3335d0fc9091121b94e2c0840b29c80c087","original_commit_id":"4fe6a3335d0fc9091121b94e2c0840b29c80c087","user":{"login":"Nikratio","id":701625,"avatar_url":"https://avatars.githubusercontent.com/u/701625?v=3","gravatar_id":"","url":"https://api.github.com/users/Nikratio","html_url":"https://github.com/Nikratio","followers_url":"https://api.github.com/users/Nikratio/followers","following_url":"https://api.github.com/users/Nikratio/following{/other_user}","gists_url":"https://api.github.com/users/Nikratio/gists{/gist_id}","starred_url":"https://api.github.com/users/Nikratio/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Nikratio/subscriptions","organizations_url":"https://api.github.com/users/Nikratio/orgs","repos_url":"https://api.github.com/users/Nikratio/repos","events_url":"https://api.github.com/users/Nikratio/events{/privacy}","received_events_url":"https://api.github.com/users/Nikratio/received_events","type":"User","site_admin":false},"body":"This needs documentation.","created_at":"2016-02-02T23:32:58Z","updated_at":"2016-02-02T23:32:58Z","html_url":"https://github.com/s3ql/s3ql/pull/4#discussion_r51655275","pull_request_url":"https://api.github.com/repos/s3ql/s3ql/pulls/4","_links":{"self":{"href":"https://api.github.com/repos/s3ql/s3ql/pulls/comments/51655275"},"html":{"href":"https://github.com/s3ql/s3ql/pull/4#discussion_r51655275"},"pull_request":{"href":"https://api.github.com/repos/s3ql/s3ql/pulls/4"}}},"pull_request":{"url":"https://api.github.com/repos/s3ql/s3ql/pulls/4","id":58038352,"html_url":"https://github.com/s3ql/s3ql/pull/4","diff_url":"https://github.com/s3ql/s3ql/pull/4.diff","patch_url":"https://github.com/s3ql/s3ql/pull/4.patch","issue_url":"https://api.github.com/repos/s3ql/s3ql/issues/4","number":4,"state":"open","locked":false,"title":"Initial implementation of Google Drive backend.","user":{"login":"mkhon","id":3022749,"avatar_url":"https://avatars.githubusercontent.com/u/3022749?v=3","gravatar_id":"","url":"https://api.github.com/users/mkhon","html_url":"https://github.com/mkhon","followers_url":"https://api.github.com/users/mkhon/followers","following_url":"https://api.github.com/users/mkhon/following{/other_user}","gists_url":"https://api.github.com/users/mkhon/gists{/gist_id}","starred_url":"https://api.github.com/users/mkhon/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mkhon/subscriptions","organizations_url":"https://api.github.com/users/mkhon/orgs","repos_url":"https://api.github.com/users/mkhon/repos","events_url":"https://api.github.com/users/mkhon/events{/privacy}","received_events_url":"https://api.github.com/users/mkhon/received_events","type":"User","site_admin":false},"body":"This is the prototype implementation.\r\n\r\n- Better error handling should probably be implemented.\r\n- ObjectR/ObjectW keep the data in-memory.","created_at":"2016-02-02T19:37:51Z","updated_at":"2016-02-02T23:32:58Z","closed_at":null,"merged_at":null,"merge_commit_sha":"2993f8bf610481a2b38d2daf8efaeb1685874435","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/s3ql/s3ql/pulls/4/commits","review_comments_url":"https://api.github.com/repos/s3ql/s3ql/pulls/4/comments","review_comment_url":"https://api.github.com/repos/s3ql/s3ql/pulls/comments{/number}","comments_url":"https://api.github.com/repos/s3ql/s3ql/issues/4/comments","statuses_url":"https://api.github.com/repos/s3ql/s3ql/statuses/4fe6a3335d0fc9091121b94e2c0840b29c80c087","head":{"label":"mkhon:gdrive","ref":"gdrive","sha":"4fe6a3335d0fc9091121b94e2c0840b29c80c087","user":{"login":"mkhon","id":3022749,"avatar_url":"https://avatars.githubusercontent.com/u/3022749?v=3","gravatar_id":"","url":"https://api.github.com/users/mkhon","html_url":"https://github.com/mkhon","followers_url":"https://api.github.com/users/mkhon/followers","following_url":"https://api.github.com/users/mkhon/following{/other_user}","gists_url":"https://api.github.com/users/mkhon/gists{/gist_id}","starred_url":"https://api.github.com/users/mkhon/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mkhon/subscriptions","organizations_url":"https://api.github.com/users/mkhon/orgs","repos_url":"https://api.github.com/users/mkhon/repos","events_url":"https://api.github.com/users/mkhon/events{/privacy}","received_events_url":"https://api.github.com/users/mkhon/received_events","type":"User","site_admin":false},"repo":{"id":48954573,"name":"s3ql","full_name":"mkhon/s3ql","owner":{"login":"mkhon","id":3022749,"avatar_url":"https://avatars.githubusercontent.com/u/3022749?v=3","gravatar_id":"","url":"https://api.github.com/users/mkhon","html_url":"https://github.com/mkhon","followers_url":"https://api.github.com/users/mkhon/followers","following_url":"https://api.github.com/users/mkhon/following{/other_user}","gists_url":"https://api.github.com/users/mkhon/gists{/gist_id}","starred_url":"https://api.github.com/users/mkhon/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mkhon/subscriptions","organizations_url":"https://api.github.com/users/mkhon/orgs","repos_url":"https://api.github.com/users/mkhon/repos","events_url":"https://api.github.com/users/mkhon/events{/privacy}","received_events_url":"https://api.github.com/users/mkhon/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/mkhon/s3ql","description":"a full featured file system for online data storage","fork":true,"url":"https://api.github.com/repos/mkhon/s3ql","forks_url":"https://api.github.com/repos/mkhon/s3ql/forks","keys_url":"https://api.github.com/repos/mkhon/s3ql/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mkhon/s3ql/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mkhon/s3ql/teams","hooks_url":"https://api.github.com/repos/mkhon/s3ql/hooks","issue_events_url":"https://api.github.com/repos/mkhon/s3ql/issues/events{/number}","events_url":"https://api.github.com/repos/mkhon/s3ql/events","assignees_url":"https://api.github.com/repos/mkhon/s3ql/assignees{/user}","branches_url":"https://api.github.com/repos/mkhon/s3ql/branches{/branch}","tags_url":"https://api.github.com/repos/mkhon/s3ql/tags","blobs_url":"https://api.github.com/repos/mkhon/s3ql/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mkhon/s3ql/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mkhon/s3ql/git/refs{/sha}","trees_url":"https://api.github.com/repos/mkhon/s3ql/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mkhon/s3ql/statuses/{sha}","languages_url":"https://api.github.com/repos/mkhon/s3ql/languages","stargazers_url":"https://api.github.com/repos/mkhon/s3ql/stargazers","contributors_url":"https://api.github.com/repos/mkhon/s3ql/contributors","subscribers_url":"https://api.github.com/repos/mkhon/s3ql/subscribers","subscription_url":"https://api.github.com/repos/mkhon/s3ql/subscription","commits_url":"https://api.github.com/repos/mkhon/s3ql/commits{/sha}","git_commits_url":"https://api.github.com/repos/mkhon/s3ql/git/commits{/sha}","comments_url":"https://api.github.com/repos/mkhon/s3ql/comments{/number}","issue_comment_url":"https://api.github.com/repos/mkhon/s3ql/issues/comments{/number}","contents_url":"https://api.github.com/repos/mkhon/s3ql/contents/{+path}","compare_url":"https://api.github.com/repos/mkhon/s3ql/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mkhon/s3ql/merges","archive_url":"https://api.github.com/repos/mkhon/s3ql/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mkhon/s3ql/downloads","issues_url":"https://api.github.com/repos/mkhon/s3ql/issues{/number}","pulls_url":"https://api.github.com/repos/mkhon/s3ql/pulls{/number}","milestones_url":"https://api.github.com/repos/mkhon/s3ql/milestones{/number}","notifications_url":"https://api.github.com/repos/mkhon/s3ql/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mkhon/s3ql/labels{/name}","releases_url":"https://api.github.com/repos/mkhon/s3ql/releases{/id}","deployments_url":"https://api.github.com/repos/mkhon/s3ql/deployments","created_at":"2016-01-03T17:31:36Z","updated_at":"2016-01-03T17:31:57Z","pushed_at":"2016-02-02T19:58:42Z","git_url":"git://github.com/mkhon/s3ql.git","ssh_url":"git@github.com:mkhon/s3ql.git","clone_url":"https://github.com/mkhon/s3ql.git","svn_url":"https://github.com/mkhon/s3ql","homepage":null,"size":6487,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"s3ql:master","ref":"master","sha":"712b14fb17187d285c4233bafe93f15bd442183d","user":{"login":"s3ql","id":13303496,"avatar_url":"https://avatars.githubusercontent.com/u/13303496?v=3","gravatar_id":"","url":"https://api.github.com/users/s3ql","html_url":"https://github.com/s3ql","followers_url":"https://api.github.com/users/s3ql/followers","following_url":"https://api.github.com/users/s3ql/following{/other_user}","gists_url":"https://api.github.com/users/s3ql/gists{/gist_id}","starred_url":"https://api.github.com/users/s3ql/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/s3ql/subscriptions","organizations_url":"https://api.github.com/users/s3ql/orgs","repos_url":"https://api.github.com/users/s3ql/repos","events_url":"https://api.github.com/users/s3ql/events{/privacy}","received_events_url":"https://api.github.com/users/s3ql/received_events","type":"Organization","site_admin":false},"repo":{"id":38976977,"name":"s3ql","full_name":"s3ql/s3ql","owner":{"login":"s3ql","id":13303496,"avatar_url":"https://avatars.githubusercontent.com/u/13303496?v=3","gravatar_id":"","url":"https://api.github.com/users/s3ql","html_url":"https://github.com/s3ql","followers_url":"https://api.github.com/users/s3ql/followers","following_url":"https://api.github.com/users/s3ql/following{/other_user}","gists_url":"https://api.github.com/users/s3ql/gists{/gist_id}","starred_url":"https://api.github.com/users/s3ql/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/s3ql/subscriptions","organizations_url":"https://api.github.com/users/s3ql/orgs","repos_url":"https://api.github.com/users/s3ql/repos","events_url":"https://api.github.com/users/s3ql/events{/privacy}","received_events_url":"https://api.github.com/users/s3ql/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/s3ql/s3ql","description":"a full featured file system for online data storage","fork":false,"url":"https://api.github.com/repos/s3ql/s3ql","forks_url":"https://api.github.com/repos/s3ql/s3ql/forks","keys_url":"https://api.github.com/repos/s3ql/s3ql/keys{/key_id}","collaborators_url":"https://api.github.com/repos/s3ql/s3ql/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/s3ql/s3ql/teams","hooks_url":"https://api.github.com/repos/s3ql/s3ql/hooks","issue_events_url":"https://api.github.com/repos/s3ql/s3ql/issues/events{/number}","events_url":"https://api.github.com/repos/s3ql/s3ql/events","assignees_url":"https://api.github.com/repos/s3ql/s3ql/assignees{/user}","branches_url":"https://api.github.com/repos/s3ql/s3ql/branches{/branch}","tags_url":"https://api.github.com/repos/s3ql/s3ql/tags","blobs_url":"https://api.github.com/repos/s3ql/s3ql/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/s3ql/s3ql/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/s3ql/s3ql/git/refs{/sha}","trees_url":"https://api.github.com/repos/s3ql/s3ql/git/trees{/sha}","statuses_url":"https://api.github.com/repos/s3ql/s3ql/statuses/{sha}","languages_url":"https://api.github.com/repos/s3ql/s3ql/languages","stargazers_url":"https://api.github.com/repos/s3ql/s3ql/stargazers","contributors_url":"https://api.github.com/repos/s3ql/s3ql/contributors","subscribers_url":"https://api.github.com/repos/s3ql/s3ql/subscribers","subscription_url":"https://api.github.com/repos/s3ql/s3ql/subscription","commits_url":"https://api.github.com/repos/s3ql/s3ql/commits{/sha}","git_commits_url":"https://api.github.com/repos/s3ql/s3ql/git/commits{/sha}","comments_url":"https://api.github.com/repos/s3ql/s3ql/comments{/number}","issue_comment_url":"https://api.github.com/repos/s3ql/s3ql/issues/comments{/number}","contents_url":"https://api.github.com/repos/s3ql/s3ql/contents/{+path}","compare_url":"https://api.github.com/repos/s3ql/s3ql/compare/{base}...{head}","merges_url":"https://api.github.com/repos/s3ql/s3ql/merges","archive_url":"https://api.github.com/repos/s3ql/s3ql/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/s3ql/s3ql/downloads","issues_url":"https://api.github.com/repos/s3ql/s3ql/issues{/number}","pulls_url":"https://api.github.com/repos/s3ql/s3ql/pulls{/number}","milestones_url":"https://api.github.com/repos/s3ql/s3ql/milestones{/number}","notifications_url":"https://api.github.com/repos/s3ql/s3ql/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/s3ql/s3ql/labels{/name}","releases_url":"https://api.github.com/repos/s3ql/s3ql/releases{/id}","deployments_url":"https://api.github.com/repos/s3ql/s3ql/deployments","created_at":"2015-07-12T20:33:18Z","updated_at":"2016-01-29T00:30:22Z","pushed_at":"2016-02-02T19:58:43Z","git_url":"git://github.com/s3ql/s3ql.git","ssh_url":"git@github.com:s3ql/s3ql.git","clone_url":"https://github.com/s3ql/s3ql.git","svn_url":"https://github.com/s3ql/s3ql","homepage":null,"size":6494,"stargazers_count":120,"watchers_count":120,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":8,"mirror_url":null,"open_issues_count":2,"forks":8,"open_issues":2,"watchers":120,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/s3ql/s3ql/pulls/4"},"html":{"href":"https://github.com/s3ql/s3ql/pull/4"},"issue":{"href":"https://api.github.com/repos/s3ql/s3ql/issues/4"},"comments":{"href":"https://api.github.com/repos/s3ql/s3ql/issues/4/comments"},"review_comments":{"href":"https://api.github.com/repos/s3ql/s3ql/pulls/4/comments"},"review_comment":{"href":"https://api.github.com/repos/s3ql/s3ql/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/s3ql/s3ql/pulls/4/commits"},"statuses":{"href":"https://api.github.com/repos/s3ql/s3ql/statuses/4fe6a3335d0fc9091121b94e2c0840b29c80c087"}}}}