{"action":"created","comment":{"url":"https://api.github.com/repos/amplifylitco/asiaq/pulls/comments/92426976","pull_request_review_id":12941623,"id":92426976,"diff_hunk":"@@ -0,0 +1,300 @@\n+\"\"\"\n+Back end behaviors for creating data pipelines.\n+\"\"\"\n+\n+import json\n+from datetime import datetime\n+from logging import getLogger\n+\n+import boto3\n+\n+from .disco_config import open_normalized\n+from .resource_helper import throttled_call\n+from .exceptions import ProgrammerError\n+\n+\n+_LOG = getLogger(__name__)\n+\n+\n+class DataPipelineConsts(object):\n+    \"\"\"Constants for data pipeline management.\"\"\"\n+    TEMPLATE_DIR = \"datapipeline_templates\"\n+\n+\n+class AsiaqDataPipeline(object):\n+    \"\"\"Local encapsulation of the information associated with a single data pipeline.\"\"\"\n+    def __init__(self, name, description, tags=None, metadata=None,\n+                 pipeline_id=None, contents=None, parameter_definitions=None, param_values=None):\n+        self._objects = contents\n+        self._params = parameter_definitions\n+        self._param_values = _optional_dict_to_list(param_values)\n+        self._name = name\n+        self._description = description\n+        self._tags = _optional_dict_to_list(tags, key_string='key', value_string='value')\n+        self._metadata = metadata\n+        self._id = pipeline_id\n+\n+    def is_persisted(self):\n+        \"Return true if this pipeline has an AWS ID; false otherwise.\"\n+        return self._id is not None\n+\n+    def has_content(self):\n+        \"Return true if this pipeline has actual pipeline objects, false if it is only metadata.\"\n+        return self._objects is not None\n+\n+    def get_tag_dict(self):\n+        \"Retrieve tags as a dictionary.\"\n+        return _optional_list_to_dict(self._tags, key_string='key', value_string='value')\n+\n+    def get_param_value_dict(self):\n+        \"Retrieve parameter values as a dictionary.\"\n+        return _optional_list_to_dict(self._param_values)\n+\n+    def update_content(self, contents=None, parameter_definitions=None, param_values=None,\n+                       template_name=None, log_location=None):\n+        \"\"\"\n+        Set the pipeline content (pipeline nodes, parameters and values) for this pipeline.\n+        \"\"\"\n+        if (not contents and not template_name) or (contents and template_name):\n+            raise ProgrammerError(\"Either pipeline content or a template (not both!) must be specified\")\n+        if template_name:\n+            contents, parameter_definitions = _read_template(template_name, log_location)\n+        self._objects = contents\n+        self._params = parameter_definitions\n+        self._param_values = _optional_dict_to_list(param_values)\n+\n+    @classmethod\n+    def from_template(cls, template_name, name, description, tags=None, param_values=None, log_location=None):\n+        \"\"\"Create a new AsiaqDataPipeline object, populated from template in the configuration directory.\"\"\"\n+        boto_objects, boto_parameters = _read_template(template_name, log_location)\n+        return cls(contents=boto_objects, parameter_definitions=boto_parameters,\n+                   name=name, description=description, tags=tags, param_values=param_values)\n+\n+\n+class AsiaqDataPipelineManager(object):\n+    \"List, retrieve, store and delete pipelines.\"\n+    def __init__(self, client=None):\n+        self._dp_client = client or boto3.client(\"datapipeline\")\n+\n+    def save(self, pipeline):\n+        \"Save or update the pipeline object in AWS.\"\n+        if not pipeline.is_persisted():  # if this is a new pipeline, save the metadata first\n+            unique_id = \"pipeline_init_%s\" % datetime.now().isoformat()  # make the create call idempotent\n+            created = throttled_call(self._dp_client.create_pipeline,\n+                                     name=pipeline._name, uniqueId=unique_id,\n+                                     description=pipeline._description, tags=pipeline._tags or [])\n+            pipeline._id = created['pipelineId']\n+        # regardless, save the pipeline content:\n+        resp = throttled_call(self._dp_client.put_pipeline_definition,\n+                              pipelineId=pipeline._id,\n+                              pipelineObjects=pipeline._objects or [],\n+                              parameterObjects=pipeline._params or [],\n+                              parameterValues=pipeline._param_values or [])\n+        return resp\n+\n+    # possibly worth having but not currently needed: set_tags\n+    # http://boto3.readthedocs.io/en/latest/reference/services/datapipeline.html#DataPipeline.Client.add_tags\n+\n+    def fetch(self, pipeline_id):\n+        \"Fetch a pipeline (metadata and content) from AWS by ID.\"\n+        contents = throttled_call(self._dp_client.get_pipeline_definition, pipelineId=pipeline_id,\n+                                  version='latest')\n+        _LOG.debug(\"Contents for %s: %s\", pipeline_id, contents)\n+        meta_resp = throttled_call(self._dp_client.describe_pipelines, pipelineIds=[pipeline_id])\n+        meta = meta_resp['pipelineDescriptionList'][0]\n+        return AsiaqDataPipeline(\n+            pipeline_id=pipeline_id, name=meta['name'], description=meta.get('description'),\n+            tags=meta.get('tags'), metadata=meta.get('fields'),\n+            contents=contents['pipelineObjects'],\n+            parameter_definitions=contents.get('parameterObjects'),\n+            param_values=contents.get('parameterValues')\n+        )\n+\n+    def fetch_content(self, pipeline):\n+        \"Populate the pipeline content fields of this pipeline object with the information fetched from AWS.\"\n+        if pipeline.has_content():\n+            raise Exception(\"Content already fetched or locally generated for this pipeline object\")","path":"disco_aws_automation/disco_datapipeline.py","position":116,"original_position":116,"commit_id":"00c268f84aa148e07e988f27c72c9c22a67dfd29","original_commit_id":"00c268f84aa148e07e988f27c72c9c22a67dfd29","user":{"login":"aoskotsky-amplify","id":15692902,"avatar_url":"https://avatars.githubusercontent.com/u/15692902?v=3","gravatar_id":"","url":"https://api.github.com/users/aoskotsky-amplify","html_url":"https://github.com/aoskotsky-amplify","followers_url":"https://api.github.com/users/aoskotsky-amplify/followers","following_url":"https://api.github.com/users/aoskotsky-amplify/following{/other_user}","gists_url":"https://api.github.com/users/aoskotsky-amplify/gists{/gist_id}","starred_url":"https://api.github.com/users/aoskotsky-amplify/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aoskotsky-amplify/subscriptions","organizations_url":"https://api.github.com/users/aoskotsky-amplify/orgs","repos_url":"https://api.github.com/users/aoskotsky-amplify/repos","events_url":"https://api.github.com/users/aoskotsky-amplify/events{/privacy}","received_events_url":"https://api.github.com/users/aoskotsky-amplify/received_events","type":"User","site_admin":false},"body":"Could we use a more specific Exception class here?","created_at":"2016-12-14T16:09:59Z","updated_at":"2016-12-14T16:09:59Z","html_url":"https://github.com/amplifylitco/asiaq/pull/182#discussion_r92426976","pull_request_url":"https://api.github.com/repos/amplifylitco/asiaq/pulls/182","_links":{"self":{"href":"https://api.github.com/repos/amplifylitco/asiaq/pulls/comments/92426976"},"html":{"href":"https://github.com/amplifylitco/asiaq/pull/182#discussion_r92426976"},"pull_request":{"href":"https://api.github.com/repos/amplifylitco/asiaq/pulls/182"}}},"pull_request":{"url":"https://api.github.com/repos/amplifylitco/asiaq/pulls/182","id":97006175,"html_url":"https://github.com/amplifylitco/asiaq/pull/182","diff_url":"https://github.com/amplifylitco/asiaq/pull/182.diff","patch_url":"https://github.com/amplifylitco/asiaq/pull/182.patch","issue_url":"https://api.github.com/repos/amplifylitco/asiaq/issues/182","number":182,"state":"open","locked":false,"title":"Data pipeline and DynamoDB backup support","user":{"login":"bwarfield-amplify","id":8418200,"avatar_url":"https://avatars.githubusercontent.com/u/8418200?v=3","gravatar_id":"","url":"https://api.github.com/users/bwarfield-amplify","html_url":"https://github.com/bwarfield-amplify","followers_url":"https://api.github.com/users/bwarfield-amplify/followers","following_url":"https://api.github.com/users/bwarfield-amplify/following{/other_user}","gists_url":"https://api.github.com/users/bwarfield-amplify/gists{/gist_id}","starred_url":"https://api.github.com/users/bwarfield-amplify/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bwarfield-amplify/subscriptions","organizations_url":"https://api.github.com/users/bwarfield-amplify/orgs","repos_url":"https://api.github.com/users/bwarfield-amplify/repos","events_url":"https://api.github.com/users/bwarfield-amplify/events{/privacy}","received_events_url":"https://api.github.com/users/bwarfield-amplify/received_events","type":"User","site_admin":false},"body":"Support for creating a data pipeline that backs up or restores a dynamodb table, listing and deleting existing pipelines, and listing existing backups.","created_at":"2016-12-07T21:01:30Z","updated_at":"2016-12-14T16:09:59Z","closed_at":null,"merged_at":null,"merge_commit_sha":"b50ce2c5846bbf2b56288e0a4ffc8a1fded8d70f","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/amplifylitco/asiaq/pulls/182/commits","review_comments_url":"https://api.github.com/repos/amplifylitco/asiaq/pulls/182/comments","review_comment_url":"https://api.github.com/repos/amplifylitco/asiaq/pulls/comments{/number}","comments_url":"https://api.github.com/repos/amplifylitco/asiaq/issues/182/comments","statuses_url":"https://api.github.com/repos/amplifylitco/asiaq/statuses/00c268f84aa148e07e988f27c72c9c22a67dfd29","head":{"label":"amplifylitco:data-pipeline-support","ref":"data-pipeline-support","sha":"00c268f84aa148e07e988f27c72c9c22a67dfd29","user":{"login":"amplifylitco","id":15659472,"avatar_url":"https://avatars.githubusercontent.com/u/15659472?v=3","gravatar_id":"","url":"https://api.github.com/users/amplifylitco","html_url":"https://github.com/amplifylitco","followers_url":"https://api.github.com/users/amplifylitco/followers","following_url":"https://api.github.com/users/amplifylitco/following{/other_user}","gists_url":"https://api.github.com/users/amplifylitco/gists{/gist_id}","starred_url":"https://api.github.com/users/amplifylitco/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amplifylitco/subscriptions","organizations_url":"https://api.github.com/users/amplifylitco/orgs","repos_url":"https://api.github.com/users/amplifylitco/repos","events_url":"https://api.github.com/users/amplifylitco/events{/privacy}","received_events_url":"https://api.github.com/users/amplifylitco/received_events","type":"Organization","site_admin":false},"repo":{"id":53361236,"name":"asiaq","full_name":"amplifylitco/asiaq","owner":{"login":"amplifylitco","id":15659472,"avatar_url":"https://avatars.githubusercontent.com/u/15659472?v=3","gravatar_id":"","url":"https://api.github.com/users/amplifylitco","html_url":"https://github.com/amplifylitco","followers_url":"https://api.github.com/users/amplifylitco/followers","following_url":"https://api.github.com/users/amplifylitco/following{/other_user}","gists_url":"https://api.github.com/users/amplifylitco/gists{/gist_id}","starred_url":"https://api.github.com/users/amplifylitco/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amplifylitco/subscriptions","organizations_url":"https://api.github.com/users/amplifylitco/orgs","repos_url":"https://api.github.com/users/amplifylitco/repos","events_url":"https://api.github.com/users/amplifylitco/events{/privacy}","received_events_url":"https://api.github.com/users/amplifylitco/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/amplifylitco/asiaq","description":null,"fork":false,"url":"https://api.github.com/repos/amplifylitco/asiaq","forks_url":"https://api.github.com/repos/amplifylitco/asiaq/forks","keys_url":"https://api.github.com/repos/amplifylitco/asiaq/keys{/key_id}","collaborators_url":"https://api.github.com/repos/amplifylitco/asiaq/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/amplifylitco/asiaq/teams","hooks_url":"https://api.github.com/repos/amplifylitco/asiaq/hooks","issue_events_url":"https://api.github.com/repos/amplifylitco/asiaq/issues/events{/number}","events_url":"https://api.github.com/repos/amplifylitco/asiaq/events","assignees_url":"https://api.github.com/repos/amplifylitco/asiaq/assignees{/user}","branches_url":"https://api.github.com/repos/amplifylitco/asiaq/branches{/branch}","tags_url":"https://api.github.com/repos/amplifylitco/asiaq/tags","blobs_url":"https://api.github.com/repos/amplifylitco/asiaq/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/amplifylitco/asiaq/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/amplifylitco/asiaq/git/refs{/sha}","trees_url":"https://api.github.com/repos/amplifylitco/asiaq/git/trees{/sha}","statuses_url":"https://api.github.com/repos/amplifylitco/asiaq/statuses/{sha}","languages_url":"https://api.github.com/repos/amplifylitco/asiaq/languages","stargazers_url":"https://api.github.com/repos/amplifylitco/asiaq/stargazers","contributors_url":"https://api.github.com/repos/amplifylitco/asiaq/contributors","subscribers_url":"https://api.github.com/repos/amplifylitco/asiaq/subscribers","subscription_url":"https://api.github.com/repos/amplifylitco/asiaq/subscription","commits_url":"https://api.github.com/repos/amplifylitco/asiaq/commits{/sha}","git_commits_url":"https://api.github.com/repos/amplifylitco/asiaq/git/commits{/sha}","comments_url":"https://api.github.com/repos/amplifylitco/asiaq/comments{/number}","issue_comment_url":"https://api.github.com/repos/amplifylitco/asiaq/issues/comments{/number}","contents_url":"https://api.github.com/repos/amplifylitco/asiaq/contents/{+path}","compare_url":"https://api.github.com/repos/amplifylitco/asiaq/compare/{base}...{head}","merges_url":"https://api.github.com/repos/amplifylitco/asiaq/merges","archive_url":"https://api.github.com/repos/amplifylitco/asiaq/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/amplifylitco/asiaq/downloads","issues_url":"https://api.github.com/repos/amplifylitco/asiaq/issues{/number}","pulls_url":"https://api.github.com/repos/amplifylitco/asiaq/pulls{/number}","milestones_url":"https://api.github.com/repos/amplifylitco/asiaq/milestones{/number}","notifications_url":"https://api.github.com/repos/amplifylitco/asiaq/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/amplifylitco/asiaq/labels{/name}","releases_url":"https://api.github.com/repos/amplifylitco/asiaq/releases{/id}","deployments_url":"https://api.github.com/repos/amplifylitco/asiaq/deployments","created_at":"2016-03-07T21:41:20Z","updated_at":"2016-11-22T11:42:25Z","pushed_at":"2016-12-14T02:01:57Z","git_url":"git://github.com/amplifylitco/asiaq.git","ssh_url":"git@github.com:amplifylitco/asiaq.git","clone_url":"https://github.com/amplifylitco/asiaq.git","svn_url":"https://github.com/amplifylitco/asiaq","homepage":null,"size":1505,"stargazers_count":12,"watchers_count":12,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":2,"mirror_url":null,"open_issues_count":19,"forks":2,"open_issues":19,"watchers":12,"default_branch":"master"}},"base":{"label":"amplifylitco:master","ref":"master","sha":"0da9df33a1960a97fb0853af647f1456e4c0b245","user":{"login":"amplifylitco","id":15659472,"avatar_url":"https://avatars.githubusercontent.com/u/15659472?v=3","gravatar_id":"","url":"https://api.github.com/users/amplifylitco","html_url":"https://github.com/amplifylitco","followers_url":"https://api.github.com/users/amplifylitco/followers","following_url":"https://api.github.com/users/amplifylitco/following{/other_user}","gists_url":"https://api.github.com/users/amplifylitco/gists{/gist_id}","starred_url":"https://api.github.com/users/amplifylitco/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amplifylitco/subscriptions","organizations_url":"https://api.github.com/users/amplifylitco/orgs","repos_url":"https://api.github.com/users/amplifylitco/repos","events_url":"https://api.github.com/users/amplifylitco/events{/privacy}","received_events_url":"https://api.github.com/users/amplifylitco/received_events","type":"Organization","site_admin":false},"repo":{"id":53361236,"name":"asiaq","full_name":"amplifylitco/asiaq","owner":{"login":"amplifylitco","id":15659472,"avatar_url":"https://avatars.githubusercontent.com/u/15659472?v=3","gravatar_id":"","url":"https://api.github.com/users/amplifylitco","html_url":"https://github.com/amplifylitco","followers_url":"https://api.github.com/users/amplifylitco/followers","following_url":"https://api.github.com/users/amplifylitco/following{/other_user}","gists_url":"https://api.github.com/users/amplifylitco/gists{/gist_id}","starred_url":"https://api.github.com/users/amplifylitco/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amplifylitco/subscriptions","organizations_url":"https://api.github.com/users/amplifylitco/orgs","repos_url":"https://api.github.com/users/amplifylitco/repos","events_url":"https://api.github.com/users/amplifylitco/events{/privacy}","received_events_url":"https://api.github.com/users/amplifylitco/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/amplifylitco/asiaq","description":null,"fork":false,"url":"https://api.github.com/repos/amplifylitco/asiaq","forks_url":"https://api.github.com/repos/amplifylitco/asiaq/forks","keys_url":"https://api.github.com/repos/amplifylitco/asiaq/keys{/key_id}","collaborators_url":"https://api.github.com/repos/amplifylitco/asiaq/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/amplifylitco/asiaq/teams","hooks_url":"https://api.github.com/repos/amplifylitco/asiaq/hooks","issue_events_url":"https://api.github.com/repos/amplifylitco/asiaq/issues/events{/number}","events_url":"https://api.github.com/repos/amplifylitco/asiaq/events","assignees_url":"https://api.github.com/repos/amplifylitco/asiaq/assignees{/user}","branches_url":"https://api.github.com/repos/amplifylitco/asiaq/branches{/branch}","tags_url":"https://api.github.com/repos/amplifylitco/asiaq/tags","blobs_url":"https://api.github.com/repos/amplifylitco/asiaq/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/amplifylitco/asiaq/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/amplifylitco/asiaq/git/refs{/sha}","trees_url":"https://api.github.com/repos/amplifylitco/asiaq/git/trees{/sha}","statuses_url":"https://api.github.com/repos/amplifylitco/asiaq/statuses/{sha}","languages_url":"https://api.github.com/repos/amplifylitco/asiaq/languages","stargazers_url":"https://api.github.com/repos/amplifylitco/asiaq/stargazers","contributors_url":"https://api.github.com/repos/amplifylitco/asiaq/contributors","subscribers_url":"https://api.github.com/repos/amplifylitco/asiaq/subscribers","subscription_url":"https://api.github.com/repos/amplifylitco/asiaq/subscription","commits_url":"https://api.github.com/repos/amplifylitco/asiaq/commits{/sha}","git_commits_url":"https://api.github.com/repos/amplifylitco/asiaq/git/commits{/sha}","comments_url":"https://api.github.com/repos/amplifylitco/asiaq/comments{/number}","issue_comment_url":"https://api.github.com/repos/amplifylitco/asiaq/issues/comments{/number}","contents_url":"https://api.github.com/repos/amplifylitco/asiaq/contents/{+path}","compare_url":"https://api.github.com/repos/amplifylitco/asiaq/compare/{base}...{head}","merges_url":"https://api.github.com/repos/amplifylitco/asiaq/merges","archive_url":"https://api.github.com/repos/amplifylitco/asiaq/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/amplifylitco/asiaq/downloads","issues_url":"https://api.github.com/repos/amplifylitco/asiaq/issues{/number}","pulls_url":"https://api.github.com/repos/amplifylitco/asiaq/pulls{/number}","milestones_url":"https://api.github.com/repos/amplifylitco/asiaq/milestones{/number}","notifications_url":"https://api.github.com/repos/amplifylitco/asiaq/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/amplifylitco/asiaq/labels{/name}","releases_url":"https://api.github.com/repos/amplifylitco/asiaq/releases{/id}","deployments_url":"https://api.github.com/repos/amplifylitco/asiaq/deployments","created_at":"2016-03-07T21:41:20Z","updated_at":"2016-11-22T11:42:25Z","pushed_at":"2016-12-14T02:01:57Z","git_url":"git://github.com/amplifylitco/asiaq.git","ssh_url":"git@github.com:amplifylitco/asiaq.git","clone_url":"https://github.com/amplifylitco/asiaq.git","svn_url":"https://github.com/amplifylitco/asiaq","homepage":null,"size":1505,"stargazers_count":12,"watchers_count":12,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":2,"mirror_url":null,"open_issues_count":19,"forks":2,"open_issues":19,"watchers":12,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/amplifylitco/asiaq/pulls/182"},"html":{"href":"https://github.com/amplifylitco/asiaq/pull/182"},"issue":{"href":"https://api.github.com/repos/amplifylitco/asiaq/issues/182"},"comments":{"href":"https://api.github.com/repos/amplifylitco/asiaq/issues/182/comments"},"review_comments":{"href":"https://api.github.com/repos/amplifylitco/asiaq/pulls/182/comments"},"review_comment":{"href":"https://api.github.com/repos/amplifylitco/asiaq/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/amplifylitco/asiaq/pulls/182/commits"},"statuses":{"href":"https://api.github.com/repos/amplifylitco/asiaq/statuses/00c268f84aa148e07e988f27c72c9c22a67dfd29"}}}}
{"actor":{"display_login":"aoskotsky-amplify"}}