{"action":"created","comment":{"url":"https://api.github.com/repos/taskcluster/ec2-manager/pulls/comments/166625124","pull_request_review_id":94717214,"id":166625124,"diff_hunk":"@@ -0,0 +1,141 @@\n+const assert = require('assert');\n+const {runAWSRequest} = require('./aws-request');\n+const Iterate = require('taskcluster-lib-iterate');\n+const log = require('./log');\n+\n+const INSTANCE_LIMIT = 200;\n+const POLL_DELAY = 30;\n+\n+class TerminationPoller {\n+  constructor({ec2, regions, runaws = runAWSRequest, pollDelay = POLL_DELAY, state}) {\n+    assert(typeof ec2 === 'object');\n+    assert(Array.isArray(regions));\n+    regions.forEach(region => assert(typeof region === 'string'));\n+    assert(typeof state === 'object');\n+\n+    this.ec2 = ec2;\n+    this.regions = regions;\n+    this.state = state;\n+    this.runaws = runaws;\n+\n+    this.iterator = new Iterate({\n+      maxIterationTime: 60 * 15,\n+      watchDog: 60 * 15,\n+      maxFailures: 10,\n+      waitTime: pollDelay,\n+      handler: async(watchdog) => {\n+        try {\n+          await this.poll(watchdog);\n+        } catch (err) {\n+          log.error({err}, 'iteration error');\n+        }\n+      },\n+    });\n+  }\n+\n+  start() {\n+    return this.iterator.start();\n+  }\n+\n+  stop() {\n+    return this.iterator.stop();\n+  }\n+\n+  async poll(watchdog, sharedState) {\n+    let toPoll = await this.state.findTerminationsToPoll(INSTANCE_LIMIT);\n+\n+    if (toPoll.length && toPoll.length === 0) {\n+      return;\n+    }\n+\n+    let resolutions = [];\n+\n+    await Promise.all(this.regions.map(async region => {\n+\n+      let instanceIds = toPoll.filter(x => x.region === region).map(x => x.id);\n+\n+      if (instanceIds.length === 0) {\n+        log.debug({region}, 'found no terminations, skipping this region');\n+        return;\n+      }\n+\n+      log.info({instanceIds, region}, 'polling terminations');\n+\n+      // Now let's ask AWS about these instances.  If the describeInstances call\n+      // fails we'll give up and try again\n+      let result;\n+      try {\n+        result = await this.runaws(this.ec2[region], 'describeInstances', {\n+          InstanceIds: instanceIds,\n+        });\n+      } catch (err) {\n+        log.warn({err}, 'failed to describe instances for termination');\n+        return;\n+      }\n+\n+      watchdog.touch();\n+\n+      let lastEvent = new Date();\n+\n+      // Now let's figure out what the code and reason are for these instances.\n+      // We're going to add these to a list of resolutions so that we can do a\n+      // single transaction later using UPDATE...ON CONFLICT... logic to\n+      // minimize DB locking\n+      for (let reservation of result.Reservations || []) {\n+        for (let instance of reservation.Instances || []) {\n+          if (instance.StateReason && instance.StateReason.Code && instance.StateReason.Message) {\n+            let m = instance.StateReason.Message;\n+            let c = instance.StateReason.Code;\n+            if (m.slice(0, c.length + 2) === c + ': ') {\n+              m = m.slice(c.length + 2);\n+            }\n+            resolutions.push({\n+              region,\n+              id: instance.InstanceId,\n+              code: c,\n+              reason: m,\n+              lastEvent,\n+            });\n+          } else {\n+            log.info({instance, region}, 'found terminated instance without code or reason');\n+          }\n+        }\n+      }\n+      watchdog.touch();\n+\n+    }));\n+\n+    // Now let's update them.  We're not using any transactions here because we\n+    // know that each instance's resolution should attempt update regardless of\n+    // whether others fail.\n+    for (let resolution of resolutions) {\n+      try {\n+        await this.state.updateTerminationState(resolution);\n+      } catch (err) {\n+        log.error({err, resolution}, 'error setting resolution reason');\n+      }\n+    }","path":"lib/termination-poller.js","position":117,"original_position":117,"commit_id":"263d682313895a70264cb56df416730ac941739f","original_commit_id":"263d682313895a70264cb56df416730ac941739f","user":{"login":"jhford","id":607353,"avatar_url":"https://avatars3.githubusercontent.com/u/607353?v=4","gravatar_id":"","url":"https://api.github.com/users/jhford","html_url":"https://github.com/jhford","followers_url":"https://api.github.com/users/jhford/followers","following_url":"https://api.github.com/users/jhford/following{/other_user}","gists_url":"https://api.github.com/users/jhford/gists{/gist_id}","starred_url":"https://api.github.com/users/jhford/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jhford/subscriptions","organizations_url":"https://api.github.com/users/jhford/orgs","repos_url":"https://api.github.com/users/jhford/repos","events_url":"https://api.github.com/users/jhford/events{/privacy}","received_events_url":"https://api.github.com/users/jhford/received_events","type":"User","site_admin":false},"body":"theoretically, sure.  These queries take ~1ms each to operate and running possibly 200 at the same time could cause the client pool to become starved for more important tasks.\r\n\r\nThe whole termination poller does not have anything close to a realtime demand here, so doing them sequentially here isn't really a concern.","created_at":"2018-02-07T14:02:38Z","updated_at":"2018-02-07T14:02:38Z","html_url":"https://github.com/taskcluster/ec2-manager/pull/56#discussion_r166625124","pull_request_url":"https://api.github.com/repos/taskcluster/ec2-manager/pulls/56","author_association":"COLLABORATOR","_links":{"self":{"href":"https://api.github.com/repos/taskcluster/ec2-manager/pulls/comments/166625124"},"html":{"href":"https://github.com/taskcluster/ec2-manager/pull/56#discussion_r166625124"},"pull_request":{"href":"https://api.github.com/repos/taskcluster/ec2-manager/pulls/56"}},"in_reply_to_id":166373719},"pull_request":{"url":"https://api.github.com/repos/taskcluster/ec2-manager/pulls/56","id":167265187,"html_url":"https://github.com/taskcluster/ec2-manager/pull/56","diff_url":"https://github.com/taskcluster/ec2-manager/pull/56.diff","patch_url":"https://github.com/taskcluster/ec2-manager/pull/56.patch","issue_url":"https://api.github.com/repos/taskcluster/ec2-manager/issues/56","number":56,"state":"open","locked":false,"title":"Many improvements","user":{"login":"jhford","id":607353,"avatar_url":"https://avatars3.githubusercontent.com/u/607353?v=4","gravatar_id":"","url":"https://api.github.com/users/jhford","html_url":"https://github.com/jhford","followers_url":"https://api.github.com/users/jhford/followers","following_url":"https://api.github.com/users/jhford/following{/other_user}","gists_url":"https://api.github.com/users/jhford/gists{/gist_id}","starred_url":"https://api.github.com/users/jhford/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jhford/subscriptions","organizations_url":"https://api.github.com/users/jhford/orgs","repos_url":"https://api.github.com/users/jhford/repos","events_url":"https://api.github.com/users/jhford/events{/privacy}","received_events_url":"https://api.github.com/users/jhford/received_events","type":"User","site_admin":false},"body":"Here's three major commits to the ec2-manager.\r\n\r\n### Polling DB for Terminations\r\nThis basically lets us know why instances shut down.  We're already able to track a couple things about instance terminations, mainly that they happen.  This patch adds a simple poller that will check every 100s (i think) for up to 200 terminations and update them in the database.\r\n\r\nThese are the error codes:\r\nhttps://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html#api-error-codes-table-server\r\n\r\nand they'll let us see why things which were running ended.  Since we're tracking instances terminate in good and bad states, as well as some metadata, we can do things like seeing whether a specific region/zone/type triplet is getting killed by amazon a lot.  We could also see things like the average time of a specific worker type.  Lots of oppurtunity for inspection!\r\n\r\nSamples from my local testing:\r\n```\r\ntesting=> select * from instances;\r\n         id          | workerType |  region   |     az     | instanceType |     state     |   imageId    |        launched        |       lastEvent        |            touched            \r\n---------------------+------------+-----------+------------+--------------+---------------+--------------+------------------------+------------------------+-------------------------------\r\n i-0e54d792b6eb7ab41 | tutorial   | us-west-1 | us-west-1c | m3.large     | shutting-down | ami-f85c1e98 | 2018-02-05 22:38:30+01 | 2018-02-05 22:40:19+01 | 2018-02-05 22:40:24.936512+01\r\n(1 row)\r\n\r\ntesting=> select * from terminations;\r\n id | workerType | region | az | instanceType | imageId | code | reason | launched | terminated | lastEvent | touched \r\n----+------------+--------+----+--------------+---------+------+--------+----------+------------+-----------+---------\r\n(0 rows)\r\n\r\ntesting=> select * from terminations;\r\n         id          | workerType |  region   |     az     | instanceType |   imageId    | code | reason |        launched        |       terminated       |       lastEvent        |            touched            \r\n---------------------+------------+-----------+------------+--------------+--------------+------+--------+------------------------+------------------------+------------------------+-------------------------------\r\n i-0e54d792b6eb7ab41 | tutorial   | us-west-1 | us-west-1c | m3.large     | ami-f85c1e98 |      |        | 2018-02-05 22:38:30+01 | 2018-02-05 22:40:53+01 | 2018-02-05 22:40:53+01 | 2018-02-05 22:40:54.715294+01\r\n(1 row)\r\n\r\ntesting=> select * from terminations;\r\n         id          | workerType |  region   |     az     | instanceType |   imageId    |             code             |         reason          |        launched        |       terminated       |         lastEvent          |            touched            \r\n---------------------+------------+-----------+------------+--------------+--------------+------------------------------+-------------------------+------------------------+------------------------+----------------------------+-------------------------------\r\n i-0e54d792b6eb7ab41 | tutorial   | us-west-1 | us-west-1c | m3.large     | ami-f85c1e98 | Client.UserInitiatedShutdown | User initiated shutdown | 2018-02-05 22:38:30+01 | 2018-02-05 22:40:53+01 | 2018-02-05 22:41:25.379+01 | 2018-02-05 22:41:25.380666+01\r\n(1 row)\r\n\r\ntesting=> \r\n\r\n```\r\n\r\n### Remove key prefix concept from ec2-manager\r\nNow that we've got atomic tag-on-create support for instances, we have no need for the ec2-manager to understand the janky key prefix system we used to use.  Yay!  '''NOTE''' that a corresponding provisioner patch will need to be written to change how SSH key creation is handled.  We might also wish to make SSH KeyPair Name a user-configurable value in the worker type definition, but with a default provisioner-wide setting.  Since this is >96h from when we deployed the tag-on-create, this should have no impact on running instances as running instances should have been created with accurate tags.\r\n\r\n### Storing of AWS Request outcomes\r\nThis is mainly intended for runInstances, but it was the same amount of work for logging all requests.  There's *tons* of information that's super handy for figuring out the state of Amazon.  We can use this information to make better choices about which region/zone/type triplets are in good shape and which are to be avoided.  Since for special cased requests (basically runInstances), I'm tracking the workerType, we'll now be able to see things like the rate at which invalid parameters are submitted to the ec2 api.  We should expose this information.\r\n\r\nSamples from my local testing:\r\n```\r\ntesting=> select * from awsrequests where method = 'runInstances';\r\n  region   |              requestId               |    duration     |    method    | service | error |           called           |         code          |                  message                   | workerType |     az     | instanceType |   imageId    \r\n-----------+--------------------------------------+-----------------+--------------+---------+-------+----------------------------+-----------------------+--------------------------------------------+------------+------------+--------------+--------------\r\n us-west-1 | 0d051dd5-3931-4b70-8fa6-bb40ea8f4bbc | 00:00:01.433223 | runInstances | ec2     | t     | 2018-02-05 22:28:43.905+01 | InvalidParameterValue | Invalid value 'z3.large' for InstanceType. | tutorial   | us-west-1c | z3.large     | ami-f85c1e98\r\n us-west-1 | 61da2107-f7ac-4407-8800-f3eb26bb09c5 | 00:00:02.140156 | runInstances | ec2     | f     | 2018-02-05 22:30:55.146+01 |                       |                                            | tutorial   | us-west-1c | m3.large     | ami-f85c1e98\r\n us-west-1 | dbc51c67-93eb-446e-bbc3-8d744efaddb0 | 00:00:01.887676 | runInstances | ec2     | f     | 2018-02-05 22:38:29.005+01 |                       |                                            | tutorial   | us-west-1c | m3.large     | ami-f85c1e98\r\n```\r\n\r\nThe aws-request stuff might look weird, because it is.  @jonasfj I submit this particular patch as an example of how annoying the AWS-SDK library is.  See the hack I need to be able to have a promise interface where there's a functional timeout and logging of request ids :)\r\n\r\nFlagging @imbstack and @walac for review\r\nCC @djmitche for thoughts on the SSH KeyPair configurability","created_at":"2018-02-05T21:55:43Z","updated_at":"2018-02-07T14:02:38Z","closed_at":null,"merged_at":null,"merge_commit_sha":"cd59033b20d8688f51ed11e5bd6d304c891fa88a","assignee":{"login":"jhford","id":607353,"avatar_url":"https://avatars3.githubusercontent.com/u/607353?v=4","gravatar_id":"","url":"https://api.github.com/users/jhford","html_url":"https://github.com/jhford","followers_url":"https://api.github.com/users/jhford/followers","following_url":"https://api.github.com/users/jhford/following{/other_user}","gists_url":"https://api.github.com/users/jhford/gists{/gist_id}","starred_url":"https://api.github.com/users/jhford/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jhford/subscriptions","organizations_url":"https://api.github.com/users/jhford/orgs","repos_url":"https://api.github.com/users/jhford/repos","events_url":"https://api.github.com/users/jhford/events{/privacy}","received_events_url":"https://api.github.com/users/jhford/received_events","type":"User","site_admin":false},"assignees":[{"login":"jhford","id":607353,"avatar_url":"https://avatars3.githubusercontent.com/u/607353?v=4","gravatar_id":"","url":"https://api.github.com/users/jhford","html_url":"https://github.com/jhford","followers_url":"https://api.github.com/users/jhford/followers","following_url":"https://api.github.com/users/jhford/following{/other_user}","gists_url":"https://api.github.com/users/jhford/gists{/gist_id}","starred_url":"https://api.github.com/users/jhford/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jhford/subscriptions","organizations_url":"https://api.github.com/users/jhford/orgs","repos_url":"https://api.github.com/users/jhford/repos","events_url":"https://api.github.com/users/jhford/events{/privacy}","received_events_url":"https://api.github.com/users/jhford/received_events","type":"User","site_admin":false}],"requested_reviewers":[],"requested_teams":[],"milestone":null,"commits_url":"https://api.github.com/repos/taskcluster/ec2-manager/pulls/56/commits","review_comments_url":"https://api.github.com/repos/taskcluster/ec2-manager/pulls/56/comments","review_comment_url":"https://api.github.com/repos/taskcluster/ec2-manager/pulls/comments{/number}","comments_url":"https://api.github.com/repos/taskcluster/ec2-manager/issues/56/comments","statuses_url":"https://api.github.com/repos/taskcluster/ec2-manager/statuses/263d682313895a70264cb56df416730ac941739f","head":{"label":"taskcluster:improvements","ref":"improvements","sha":"263d682313895a70264cb56df416730ac941739f","user":{"login":"taskcluster","id":6257436,"avatar_url":"https://avatars2.githubusercontent.com/u/6257436?v=4","gravatar_id":"","url":"https://api.github.com/users/taskcluster","html_url":"https://github.com/taskcluster","followers_url":"https://api.github.com/users/taskcluster/followers","following_url":"https://api.github.com/users/taskcluster/following{/other_user}","gists_url":"https://api.github.com/users/taskcluster/gists{/gist_id}","starred_url":"https://api.github.com/users/taskcluster/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/taskcluster/subscriptions","organizations_url":"https://api.github.com/users/taskcluster/orgs","repos_url":"https://api.github.com/users/taskcluster/repos","events_url":"https://api.github.com/users/taskcluster/events{/privacy}","received_events_url":"https://api.github.com/users/taskcluster/received_events","type":"Organization","site_admin":false},"repo":{"id":89593937,"name":"ec2-manager","full_name":"taskcluster/ec2-manager","owner":{"login":"taskcluster","id":6257436,"avatar_url":"https://avatars2.githubusercontent.com/u/6257436?v=4","gravatar_id":"","url":"https://api.github.com/users/taskcluster","html_url":"https://github.com/taskcluster","followers_url":"https://api.github.com/users/taskcluster/followers","following_url":"https://api.github.com/users/taskcluster/following{/other_user}","gists_url":"https://api.github.com/users/taskcluster/gists{/gist_id}","starred_url":"https://api.github.com/users/taskcluster/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/taskcluster/subscriptions","organizations_url":"https://api.github.com/users/taskcluster/orgs","repos_url":"https://api.github.com/users/taskcluster/repos","events_url":"https://api.github.com/users/taskcluster/events{/privacy}","received_events_url":"https://api.github.com/users/taskcluster/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/taskcluster/ec2-manager","description":null,"fork":false,"url":"https://api.github.com/repos/taskcluster/ec2-manager","forks_url":"https://api.github.com/repos/taskcluster/ec2-manager/forks","keys_url":"https://api.github.com/repos/taskcluster/ec2-manager/keys{/key_id}","collaborators_url":"https://api.github.com/repos/taskcluster/ec2-manager/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/taskcluster/ec2-manager/teams","hooks_url":"https://api.github.com/repos/taskcluster/ec2-manager/hooks","issue_events_url":"https://api.github.com/repos/taskcluster/ec2-manager/issues/events{/number}","events_url":"https://api.github.com/repos/taskcluster/ec2-manager/events","assignees_url":"https://api.github.com/repos/taskcluster/ec2-manager/assignees{/user}","branches_url":"https://api.github.com/repos/taskcluster/ec2-manager/branches{/branch}","tags_url":"https://api.github.com/repos/taskcluster/ec2-manager/tags","blobs_url":"https://api.github.com/repos/taskcluster/ec2-manager/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/taskcluster/ec2-manager/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/taskcluster/ec2-manager/git/refs{/sha}","trees_url":"https://api.github.com/repos/taskcluster/ec2-manager/git/trees{/sha}","statuses_url":"https://api.github.com/repos/taskcluster/ec2-manager/statuses/{sha}","languages_url":"https://api.github.com/repos/taskcluster/ec2-manager/languages","stargazers_url":"https://api.github.com/repos/taskcluster/ec2-manager/stargazers","contributors_url":"https://api.github.com/repos/taskcluster/ec2-manager/contributors","subscribers_url":"https://api.github.com/repos/taskcluster/ec2-manager/subscribers","subscription_url":"https://api.github.com/repos/taskcluster/ec2-manager/subscription","commits_url":"https://api.github.com/repos/taskcluster/ec2-manager/commits{/sha}","git_commits_url":"https://api.github.com/repos/taskcluster/ec2-manager/git/commits{/sha}","comments_url":"https://api.github.com/repos/taskcluster/ec2-manager/comments{/number}","issue_comment_url":"https://api.github.com/repos/taskcluster/ec2-manager/issues/comments{/number}","contents_url":"https://api.github.com/repos/taskcluster/ec2-manager/contents/{+path}","compare_url":"https://api.github.com/repos/taskcluster/ec2-manager/compare/{base}...{head}","merges_url":"https://api.github.com/repos/taskcluster/ec2-manager/merges","archive_url":"https://api.github.com/repos/taskcluster/ec2-manager/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/taskcluster/ec2-manager/downloads","issues_url":"https://api.github.com/repos/taskcluster/ec2-manager/issues{/number}","pulls_url":"https://api.github.com/repos/taskcluster/ec2-manager/pulls{/number}","milestones_url":"https://api.github.com/repos/taskcluster/ec2-manager/milestones{/number}","notifications_url":"https://api.github.com/repos/taskcluster/ec2-manager/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/taskcluster/ec2-manager/labels{/name}","releases_url":"https://api.github.com/repos/taskcluster/ec2-manager/releases{/id}","deployments_url":"https://api.github.com/repos/taskcluster/ec2-manager/deployments","created_at":"2017-04-27T12:16:36Z","updated_at":"2017-10-04T19:05:53Z","pushed_at":"2018-02-07T13:29:14Z","git_url":"git://github.com/taskcluster/ec2-manager.git","ssh_url":"git@github.com:taskcluster/ec2-manager.git","clone_url":"https://github.com/taskcluster/ec2-manager.git","svn_url":"https://github.com/taskcluster/ec2-manager","homepage":null,"size":380,"stargazers_count":3,"watchers_count":3,"language":"JavaScript","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":10,"mirror_url":null,"archived":false,"open_issues_count":20,"license":null,"forks":10,"open_issues":20,"watchers":3,"default_branch":"master"}},"base":{"label":"taskcluster:master","ref":"master","sha":"02b06fee93a67318698d2cea2ceab929dd0f654f","user":{"login":"taskcluster","id":6257436,"avatar_url":"https://avatars2.githubusercontent.com/u/6257436?v=4","gravatar_id":"","url":"https://api.github.com/users/taskcluster","html_url":"https://github.com/taskcluster","followers_url":"https://api.github.com/users/taskcluster/followers","following_url":"https://api.github.com/users/taskcluster/following{/other_user}","gists_url":"https://api.github.com/users/taskcluster/gists{/gist_id}","starred_url":"https://api.github.com/users/taskcluster/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/taskcluster/subscriptions","organizations_url":"https://api.github.com/users/taskcluster/orgs","repos_url":"https://api.github.com/users/taskcluster/repos","events_url":"https://api.github.com/users/taskcluster/events{/privacy}","received_events_url":"https://api.github.com/users/taskcluster/received_events","type":"Organization","site_admin":false},"repo":{"id":89593937,"name":"ec2-manager","full_name":"taskcluster/ec2-manager","owner":{"login":"taskcluster","id":6257436,"avatar_url":"https://avatars2.githubusercontent.com/u/6257436?v=4","gravatar_id":"","url":"https://api.github.com/users/taskcluster","html_url":"https://github.com/taskcluster","followers_url":"https://api.github.com/users/taskcluster/followers","following_url":"https://api.github.com/users/taskcluster/following{/other_user}","gists_url":"https://api.github.com/users/taskcluster/gists{/gist_id}","starred_url":"https://api.github.com/users/taskcluster/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/taskcluster/subscriptions","organizations_url":"https://api.github.com/users/taskcluster/orgs","repos_url":"https://api.github.com/users/taskcluster/repos","events_url":"https://api.github.com/users/taskcluster/events{/privacy}","received_events_url":"https://api.github.com/users/taskcluster/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/taskcluster/ec2-manager","description":null,"fork":false,"url":"https://api.github.com/repos/taskcluster/ec2-manager","forks_url":"https://api.github.com/repos/taskcluster/ec2-manager/forks","keys_url":"https://api.github.com/repos/taskcluster/ec2-manager/keys{/key_id}","collaborators_url":"https://api.github.com/repos/taskcluster/ec2-manager/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/taskcluster/ec2-manager/teams","hooks_url":"https://api.github.com/repos/taskcluster/ec2-manager/hooks","issue_events_url":"https://api.github.com/repos/taskcluster/ec2-manager/issues/events{/number}","events_url":"https://api.github.com/repos/taskcluster/ec2-manager/events","assignees_url":"https://api.github.com/repos/taskcluster/ec2-manager/assignees{/user}","branches_url":"https://api.github.com/repos/taskcluster/ec2-manager/branches{/branch}","tags_url":"https://api.github.com/repos/taskcluster/ec2-manager/tags","blobs_url":"https://api.github.com/repos/taskcluster/ec2-manager/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/taskcluster/ec2-manager/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/taskcluster/ec2-manager/git/refs{/sha}","trees_url":"https://api.github.com/repos/taskcluster/ec2-manager/git/trees{/sha}","statuses_url":"https://api.github.com/repos/taskcluster/ec2-manager/statuses/{sha}","languages_url":"https://api.github.com/repos/taskcluster/ec2-manager/languages","stargazers_url":"https://api.github.com/repos/taskcluster/ec2-manager/stargazers","contributors_url":"https://api.github.com/repos/taskcluster/ec2-manager/contributors","subscribers_url":"https://api.github.com/repos/taskcluster/ec2-manager/subscribers","subscription_url":"https://api.github.com/repos/taskcluster/ec2-manager/subscription","commits_url":"https://api.github.com/repos/taskcluster/ec2-manager/commits{/sha}","git_commits_url":"https://api.github.com/repos/taskcluster/ec2-manager/git/commits{/sha}","comments_url":"https://api.github.com/repos/taskcluster/ec2-manager/comments{/number}","issue_comment_url":"https://api.github.com/repos/taskcluster/ec2-manager/issues/comments{/number}","contents_url":"https://api.github.com/repos/taskcluster/ec2-manager/contents/{+path}","compare_url":"https://api.github.com/repos/taskcluster/ec2-manager/compare/{base}...{head}","merges_url":"https://api.github.com/repos/taskcluster/ec2-manager/merges","archive_url":"https://api.github.com/repos/taskcluster/ec2-manager/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/taskcluster/ec2-manager/downloads","issues_url":"https://api.github.com/repos/taskcluster/ec2-manager/issues{/number}","pulls_url":"https://api.github.com/repos/taskcluster/ec2-manager/pulls{/number}","milestones_url":"https://api.github.com/repos/taskcluster/ec2-manager/milestones{/number}","notifications_url":"https://api.github.com/repos/taskcluster/ec2-manager/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/taskcluster/ec2-manager/labels{/name}","releases_url":"https://api.github.com/repos/taskcluster/ec2-manager/releases{/id}","deployments_url":"https://api.github.com/repos/taskcluster/ec2-manager/deployments","created_at":"2017-04-27T12:16:36Z","updated_at":"2017-10-04T19:05:53Z","pushed_at":"2018-02-07T13:29:14Z","git_url":"git://github.com/taskcluster/ec2-manager.git","ssh_url":"git@github.com:taskcluster/ec2-manager.git","clone_url":"https://github.com/taskcluster/ec2-manager.git","svn_url":"https://github.com/taskcluster/ec2-manager","homepage":null,"size":380,"stargazers_count":3,"watchers_count":3,"language":"JavaScript","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":10,"mirror_url":null,"archived":false,"open_issues_count":20,"license":null,"forks":10,"open_issues":20,"watchers":3,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/taskcluster/ec2-manager/pulls/56"},"html":{"href":"https://github.com/taskcluster/ec2-manager/pull/56"},"issue":{"href":"https://api.github.com/repos/taskcluster/ec2-manager/issues/56"},"comments":{"href":"https://api.github.com/repos/taskcluster/ec2-manager/issues/56/comments"},"review_comments":{"href":"https://api.github.com/repos/taskcluster/ec2-manager/pulls/56/comments"},"review_comment":{"href":"https://api.github.com/repos/taskcluster/ec2-manager/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/taskcluster/ec2-manager/pulls/56/commits"},"statuses":{"href":"https://api.github.com/repos/taskcluster/ec2-manager/statuses/263d682313895a70264cb56df416730ac941739f"}},"author_association":"COLLABORATOR"}}
{"actor":{"display_login":"jhford"}}