{"action":"created","comment":{"url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/comments/78452067","id":78452067,"diff_hunk":"@@ -0,0 +1,166 @@\n+\"\"\"A simple client for authenticated access to Open edX REST APIs.\"\"\"\n+\n+from datetime import datetime, timedelta\n+import logging\n+\n+from luigi import configuration\n+import requests\n+from requests.auth import AuthBase\n+\n+from edx.analytics.tasks.util.retry import retry\n+\n+\n+log = logging.getLogger(__name__)\n+\n+DEFAULT_RETRY_STATUS_CODES = (\n+    requests.codes.request_timeout,\n+    requests.codes.too_many_requests,\n+    requests.codes.service_unavailable,\n+    requests.codes.gateway_timeout\n+)\n+DEFAULT_TIMEOUT = 7200\n+\n+\n+class EdxApiClient(object):\n+    \"\"\"\n+    Simplifies authentication and pagination logic when communicating with Open edX REST APIs.\n+\n+    Note that we deliberately chose *not* to use the edx-rest-api-client here since slumber was mostly just getting in\n+    the way. There is a small amount of code duplication between this client and that one.\n+\n+    Sample configuration that can be included in the configuration file:\n+\n+    [edx-rest-api]\n+    auth_url = https://example.com/oauth2/v1/access_token\n+    client_id = 1234567890\n+    client_secret = this-should-be-a-secret\n+\n+    Arguments:\n+        auth_url (str): The full URL of of the authentication endpoint. It should look like\n+            https://example.com/oauth2/v1/access_token. If this is not provided it is read from the \"auth_url\" field\n+            of the \"edx-rest-api\" section of the configuration file.\n+        client_id (str): This is the OAuth2 client_id for the application. If it is not provided it is read from the\n+            \"client_id\" field of the \"edx-rest-api\" section of the configuration file.\n+        client_secret (str): This is the OAuth2 client_secret for the application. If it is not provided it is read from\n+            the \"client_secret\" field of the \"edx-rest-api\" section of the configuration file.\n+    \"\"\"\n+\n+    def __init__(self, auth_url=None, client_id=None, client_secret=None):\n+        self._expires_at = None\n+        self._session = requests.Session()\n+        self._session.hooks = {\n+            'response': log_response_hook\n+        }\n+\n+        config = configuration.get_config()\n+        self.client_id = client_id or config.get('edx-rest-api', 'client_id')\n+        self.client_secret = client_secret or config.get('edx-rest-api', 'client_secret')\n+        self.auth_url = auth_url or config.get('edx-rest-api', 'auth_url')\n+\n+    @property\n+    def authenticated_session(self):\n+        \"\"\"A session that has a valid access token associated with it and can make authenticated requests.\"\"\"\n+        self.ensure_oauth_access_token()\n+        return self._session\n+\n+    def ensure_oauth_access_token(self):\n+        \"\"\"Retrieves OAuth 2.0 access token using the client credentials grant and stores it in the request session.\"\"\"\n+        now = datetime.utcnow()\n+        if self._expires_at is None or now >= self._expires_at:\n+            log.info('Token is expired or missing, requesting a new one.')\n+            response = requests.post(\n+                self.auth_url,\n+                data={\n+                    'grant_type': 'client_credentials',\n+                    'client_id': self.client_id,\n+                    'client_secret': self.client_secret,\n+                    'token_type': 'jwt',\n+                },\n+                hooks={\n+                    'response': log_response_hook\n+                }\n+            )\n+            data = response.json()\n+            self._session.auth = SuppliedJwtAuth(data['access_token'])\n+            self._expires_at = now + timedelta(seconds=data['expires_in'])\n+            log.info('Acquired a token that expires at %s', self._expires_at.isoformat())\n+\n+    def paginated_get(self, url, params=None, timeout_seconds=DEFAULT_TIMEOUT,\n+                      retry_on=DEFAULT_RETRY_STATUS_CODES, pagination_key='next'):\n+        \"\"\"\n+        Fetches a paginated resource.\n+\n+        Arguments:\n+            url (str): The URL of the resource.\n+            params (dict): This is a dictionary of key-value pairs that are URL encoded and injected into the query\n+                string when making the request.\n+            timeout_seconds (float): When requesting a page, keep retrying unless this much time has elapsed. Raise an\n+                error if a successful response cannot be fetched in this amount of time.","path":"edx/analytics/tasks/util/edx_api_client.py","position":98,"original_position":98,"commit_id":"1158c604b962a89fcab50daade0e077f1ed0cbf9","original_commit_id":"1158c604b962a89fcab50daade0e077f1ed0cbf9","user":{"login":"mulby","id":627177,"avatar_url":"https://avatars.githubusercontent.com/u/627177?v=3","gravatar_id":"","url":"https://api.github.com/users/mulby","html_url":"https://github.com/mulby","followers_url":"https://api.github.com/users/mulby/followers","following_url":"https://api.github.com/users/mulby/following{/other_user}","gists_url":"https://api.github.com/users/mulby/gists{/gist_id}","starred_url":"https://api.github.com/users/mulby/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mulby/subscriptions","organizations_url":"https://api.github.com/users/mulby/orgs","repos_url":"https://api.github.com/users/mulby/repos","events_url":"https://api.github.com/users/mulby/events{/privacy}","received_events_url":"https://api.github.com/users/mulby/received_events","type":"User","site_admin":false},"body":"I had hoped to convey that, but now see that it's still somewhat ambiguous. I've tried to clarify the wording.","created_at":"2016-09-12T20:51:05Z","updated_at":"2016-09-12T20:51:05Z","html_url":"https://github.com/edx/edx-analytics-pipeline/pull/283#discussion_r78452067","pull_request_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/283","_links":{"self":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/comments/78452067"},"html":{"href":"https://github.com/edx/edx-analytics-pipeline/pull/283#discussion_r78452067"},"pull_request":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/283"}}},"pull_request":{"url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/283","id":82573558,"html_url":"https://github.com/edx/edx-analytics-pipeline/pull/283","diff_url":"https://github.com/edx/edx-analytics-pipeline/pull/283.diff","patch_url":"https://github.com/edx/edx-analytics-pipeline/pull/283.patch","issue_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/283","number":283,"state":"open","locked":false,"title":"Create a program mapping table in the data warehouse","user":{"login":"mulby","id":627177,"avatar_url":"https://avatars.githubusercontent.com/u/627177?v=3","gravatar_id":"","url":"https://api.github.com/users/mulby","html_url":"https://github.com/mulby","followers_url":"https://api.github.com/users/mulby/followers","following_url":"https://api.github.com/users/mulby/following{/other_user}","gists_url":"https://api.github.com/users/mulby/gists{/gist_id}","starred_url":"https://api.github.com/users/mulby/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mulby/subscriptions","organizations_url":"https://api.github.com/users/mulby/orgs","repos_url":"https://api.github.com/users/mulby/repos","events_url":"https://api.github.com/users/mulby/events{/privacy}","received_events_url":"https://api.github.com/users/mulby/received_events","type":"User","site_admin":false},"body":"","created_at":"2016-08-24T16:18:12Z","updated_at":"2016-09-12T20:51:05Z","closed_at":null,"merged_at":null,"merge_commit_sha":"2f4d11093105ce181a6398521881f2c67faaf145","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/283/commits","review_comments_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/283/comments","review_comment_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/comments{/number}","comments_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/283/comments","statuses_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/statuses/1158c604b962a89fcab50daade0e077f1ed0cbf9","head":{"label":"edx:gabe/use-course-discovery","ref":"gabe/use-course-discovery","sha":"1158c604b962a89fcab50daade0e077f1ed0cbf9","user":{"login":"edx","id":3179841,"avatar_url":"https://avatars.githubusercontent.com/u/3179841?v=3","gravatar_id":"","url":"https://api.github.com/users/edx","html_url":"https://github.com/edx","followers_url":"https://api.github.com/users/edx/followers","following_url":"https://api.github.com/users/edx/following{/other_user}","gists_url":"https://api.github.com/users/edx/gists{/gist_id}","starred_url":"https://api.github.com/users/edx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edx/subscriptions","organizations_url":"https://api.github.com/users/edx/orgs","repos_url":"https://api.github.com/users/edx/repos","events_url":"https://api.github.com/users/edx/events{/privacy}","received_events_url":"https://api.github.com/users/edx/received_events","type":"Organization","site_admin":false},"repo":{"id":22929261,"name":"edx-analytics-pipeline","full_name":"edx/edx-analytics-pipeline","owner":{"login":"edx","id":3179841,"avatar_url":"https://avatars.githubusercontent.com/u/3179841?v=3","gravatar_id":"","url":"https://api.github.com/users/edx","html_url":"https://github.com/edx","followers_url":"https://api.github.com/users/edx/followers","following_url":"https://api.github.com/users/edx/following{/other_user}","gists_url":"https://api.github.com/users/edx/gists{/gist_id}","starred_url":"https://api.github.com/users/edx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edx/subscriptions","organizations_url":"https://api.github.com/users/edx/orgs","repos_url":"https://api.github.com/users/edx/repos","events_url":"https://api.github.com/users/edx/events{/privacy}","received_events_url":"https://api.github.com/users/edx/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/edx/edx-analytics-pipeline","description":"","fork":false,"url":"https://api.github.com/repos/edx/edx-analytics-pipeline","forks_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/forks","keys_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/keys{/key_id}","collaborators_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/teams","hooks_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/hooks","issue_events_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/events{/number}","events_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/events","assignees_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/assignees{/user}","branches_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/branches{/branch}","tags_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/tags","blobs_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/refs{/sha}","trees_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/trees{/sha}","statuses_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/statuses/{sha}","languages_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/languages","stargazers_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/stargazers","contributors_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/contributors","subscribers_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/subscribers","subscription_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/subscription","commits_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/commits{/sha}","git_commits_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/commits{/sha}","comments_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/comments{/number}","issue_comment_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/comments{/number}","contents_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/contents/{+path}","compare_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/compare/{base}...{head}","merges_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/merges","archive_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/downloads","issues_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues{/number}","pulls_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls{/number}","milestones_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/milestones{/number}","notifications_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/labels{/name}","releases_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/releases{/id}","deployments_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/deployments","created_at":"2014-08-13T19:57:17Z","updated_at":"2016-09-12T11:38:13Z","pushed_at":"2016-09-12T18:11:14Z","git_url":"git://github.com/edx/edx-analytics-pipeline.git","ssh_url":"git@github.com:edx/edx-analytics-pipeline.git","clone_url":"https://github.com/edx/edx-analytics-pipeline.git","svn_url":"https://github.com/edx/edx-analytics-pipeline","homepage":null,"size":4187,"stargazers_count":28,"watchers_count":28,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":45,"mirror_url":null,"open_issues_count":13,"forks":45,"open_issues":13,"watchers":28,"default_branch":"master"}},"base":{"label":"edx:master","ref":"master","sha":"3b79e916b3a30f92e8e18f4851b1917b6cf1d2ae","user":{"login":"edx","id":3179841,"avatar_url":"https://avatars.githubusercontent.com/u/3179841?v=3","gravatar_id":"","url":"https://api.github.com/users/edx","html_url":"https://github.com/edx","followers_url":"https://api.github.com/users/edx/followers","following_url":"https://api.github.com/users/edx/following{/other_user}","gists_url":"https://api.github.com/users/edx/gists{/gist_id}","starred_url":"https://api.github.com/users/edx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edx/subscriptions","organizations_url":"https://api.github.com/users/edx/orgs","repos_url":"https://api.github.com/users/edx/repos","events_url":"https://api.github.com/users/edx/events{/privacy}","received_events_url":"https://api.github.com/users/edx/received_events","type":"Organization","site_admin":false},"repo":{"id":22929261,"name":"edx-analytics-pipeline","full_name":"edx/edx-analytics-pipeline","owner":{"login":"edx","id":3179841,"avatar_url":"https://avatars.githubusercontent.com/u/3179841?v=3","gravatar_id":"","url":"https://api.github.com/users/edx","html_url":"https://github.com/edx","followers_url":"https://api.github.com/users/edx/followers","following_url":"https://api.github.com/users/edx/following{/other_user}","gists_url":"https://api.github.com/users/edx/gists{/gist_id}","starred_url":"https://api.github.com/users/edx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edx/subscriptions","organizations_url":"https://api.github.com/users/edx/orgs","repos_url":"https://api.github.com/users/edx/repos","events_url":"https://api.github.com/users/edx/events{/privacy}","received_events_url":"https://api.github.com/users/edx/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/edx/edx-analytics-pipeline","description":"","fork":false,"url":"https://api.github.com/repos/edx/edx-analytics-pipeline","forks_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/forks","keys_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/keys{/key_id}","collaborators_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/teams","hooks_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/hooks","issue_events_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/events{/number}","events_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/events","assignees_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/assignees{/user}","branches_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/branches{/branch}","tags_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/tags","blobs_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/refs{/sha}","trees_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/trees{/sha}","statuses_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/statuses/{sha}","languages_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/languages","stargazers_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/stargazers","contributors_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/contributors","subscribers_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/subscribers","subscription_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/subscription","commits_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/commits{/sha}","git_commits_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/git/commits{/sha}","comments_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/comments{/number}","issue_comment_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/comments{/number}","contents_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/contents/{+path}","compare_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/compare/{base}...{head}","merges_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/merges","archive_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/downloads","issues_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues{/number}","pulls_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls{/number}","milestones_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/milestones{/number}","notifications_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/labels{/name}","releases_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/releases{/id}","deployments_url":"https://api.github.com/repos/edx/edx-analytics-pipeline/deployments","created_at":"2014-08-13T19:57:17Z","updated_at":"2016-09-12T11:38:13Z","pushed_at":"2016-09-12T18:11:14Z","git_url":"git://github.com/edx/edx-analytics-pipeline.git","ssh_url":"git@github.com:edx/edx-analytics-pipeline.git","clone_url":"https://github.com/edx/edx-analytics-pipeline.git","svn_url":"https://github.com/edx/edx-analytics-pipeline","homepage":null,"size":4187,"stargazers_count":28,"watchers_count":28,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":45,"mirror_url":null,"open_issues_count":13,"forks":45,"open_issues":13,"watchers":28,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/283"},"html":{"href":"https://github.com/edx/edx-analytics-pipeline/pull/283"},"issue":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/283"},"comments":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/issues/283/comments"},"review_comments":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/283/comments"},"review_comment":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/pulls/283/commits"},"statuses":{"href":"https://api.github.com/repos/edx/edx-analytics-pipeline/statuses/1158c604b962a89fcab50daade0e077f1ed0cbf9"}}}}
{"actor":{"display_login":"mulby"}}