{"action":"created","comment":{"url":"https://api.github.com/repos/dimagi/commcare-cloud/pulls/comments/168303231","pull_request_review_id":96649099,"id":168303231,"diff_hunk":"@@ -38,73 +49,112 @@ class AwsConfig(jsonobject.JsonObject):\n     subnet = jsonobject.StringProperty()\n \n \n-class Allocation(jsonobject.JsonObject):\n+class Allocation(StrictJsonObject):\n     count = jsonobject.IntegerProperty()\n     from_ = jsonobject.StringProperty(name='from')\n \n \n-class AnsibleInventory(jsonobject.JsonObject):\n-    all = jsonobject.ObjectProperty(lambda: AnsibleInventoryGroup)\n+# Inventory\n+\n+class Inventory(StrictJsonObject):\n+    all_hosts = jsonobject.ListProperty(lambda: Host)\n+    all_groups = jsonobject.DictProperty(lambda: Group)\n+\n \n+class Host(StrictJsonObject):\n+    name = jsonobject.StringProperty()\n+    public_ip = jsonobject.StringProperty()\n+    private_ip = jsonobject.StringProperty()\n+    vars = jsonobject.DictProperty()\n \n-class AnsibleInventoryGroup(jsonobject.JsonObject):\n-    children = jsonobject.DictProperty(lambda: AnsibleInventoryGroup, required=False)\n-    # values can be var:value dicts, but are often null\n-    hosts = jsonobject.DictProperty(jsonobject.DictProperty(), exclude_if_none=True)\n+\n+class Group(StrictJsonObject):\n+    name = jsonobject.StringProperty()\n+    host_names = jsonobject.ListProperty(unicode)\n+    vars = jsonobject.DictProperty()\n \n \n def provision_machines(spec, env=None):\n     if env is None:\n         env = u'hq-{}'.format(\n-            ''.join(random.choice(string.ascii_lowercase + string.digits) for i in range(7))\n+            ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(7))\n         )\n-    all_hosts, inventory = bootstrap_inventory(spec, env)\n-    instance_ids = ask_aws_for_instances(env, spec.aws_config, len(all_hosts))\n+    inventory = bootstrap_inventory(spec, env)\n+    instance_ids = ask_aws_for_instances(env, spec.aws_config, len(inventory.all_hosts))\n \n     while True:\n         instance_ip_addresses = poll_for_aws_state(env, instance_ids)\n         if instance_ip_addresses:\n             break\n \n-    ip_addresses_by_host = dict(zip(all_hosts, instance_ip_addresses.values()))\n+    hosts_by_name = {}\n+\n+    for host, (public_ip, private_ip) in zip(inventory.all_hosts, instance_ip_addresses.values()):\n+        host.public_ip = public_ip\n+        host.private_ip = private_ip\n+        hosts_by_name[host.name] = host\n \n-    for group_name, group in inventory.all.children.items():\n-        for host_name, host_vars in group.hosts.items():\n-            host_vars['ansible_host'] = str(ip_addresses_by_host[host_name])\n+    for i, host_name in enumerate(inventory.all_groups['kafka'].host_names):\n+        hosts_by_name[host_name].vars['kafka_broker_id'] = i\n+        hosts_by_name[host_name].vars['swap_size'] = '2G'\n+\n+    for host_name in inventory.all_groups['elasticsearch'].host_names:\n+        hosts_by_name[host_name].vars['elasticsearch_node_name'] = host_name\n \n     save_inventory(env, inventory)\n     copy_default_vars(env, spec.aws_config)\n \n \n+def alphanumeric_sort_key(key):\n+    \"\"\"\n+    Sort the given iterable in the way that humans expect.\n+    Thanks to http://stackoverflow.com/a/2669120/240553\n+    \"\"\"\n+    import re\n+    convert = lambda text: int(text) if text.isdigit() else text","path":"commcare-cloud-bootstrap/commcare_cloud_bootstrap.py","position":118,"original_position":118,"commit_id":"1d1d4157482b3f3171d534895a8e4a7e47ec389a","original_commit_id":"1d1d4157482b3f3171d534895a8e4a7e47ec389a","user":{"login":"dannyroberts","id":137212,"avatar_url":"https://avatars1.githubusercontent.com/u/137212?v=4","gravatar_id":"","url":"https://api.github.com/users/dannyroberts","html_url":"https://github.com/dannyroberts","followers_url":"https://api.github.com/users/dannyroberts/followers","following_url":"https://api.github.com/users/dannyroberts/following{/other_user}","gists_url":"https://api.github.com/users/dannyroberts/gists{/gist_id}","starred_url":"https://api.github.com/users/dannyroberts/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dannyroberts/subscriptions","organizations_url":"https://api.github.com/users/dannyroberts/orgs","repos_url":"https://api.github.com/users/dannyroberts/repos","events_url":"https://api.github.com/users/dannyroberts/events{/privacy}","received_events_url":"https://api.github.com/users/dannyroberts/received_events","type":"User","site_admin":false},"body":"Yeah, but I didn't have to worry about that for what we're doing here because what I'm sorting is always lowercase.","created_at":"2018-02-14T20:40:21Z","updated_at":"2018-02-14T20:40:22Z","html_url":"https://github.com/dimagi/commcare-cloud/pull/1348#discussion_r168303231","pull_request_url":"https://api.github.com/repos/dimagi/commcare-cloud/pulls/1348","author_association":"OWNER","_links":{"self":{"href":"https://api.github.com/repos/dimagi/commcare-cloud/pulls/comments/168303231"},"html":{"href":"https://github.com/dimagi/commcare-cloud/pull/1348#discussion_r168303231"},"pull_request":{"href":"https://api.github.com/repos/dimagi/commcare-cloud/pulls/1348"}},"in_reply_to_id":168290548},"pull_request":{"url":"https://api.github.com/repos/dimagi/commcare-cloud/pulls/1348","id":168952282,"html_url":"https://github.com/dimagi/commcare-cloud/pull/1348","diff_url":"https://github.com/dimagi/commcare-cloud/pull/1348.diff","patch_url":"https://github.com/dimagi/commcare-cloud/pull/1348.patch","issue_url":"https://api.github.com/repos/dimagi/commcare-cloud/issues/1348","number":1348,"state":"open","locked":false,"title":"Continue work on commcare-cloud-bootstrap","user":{"login":"dannyroberts","id":137212,"avatar_url":"https://avatars1.githubusercontent.com/u/137212?v=4","gravatar_id":"","url":"https://api.github.com/users/dannyroberts","html_url":"https://github.com/dannyroberts","followers_url":"https://api.github.com/users/dannyroberts/followers","following_url":"https://api.github.com/users/dannyroberts/following{/other_user}","gists_url":"https://api.github.com/users/dannyroberts/gists{/gist_id}","starred_url":"https://api.github.com/users/dannyroberts/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dannyroberts/subscriptions","organizations_url":"https://api.github.com/users/dannyroberts/orgs","repos_url":"https://api.github.com/users/dannyroberts/repos","events_url":"https://api.github.com/users/dannyroberts/events{/privacy}","received_events_url":"https://api.github.com/users/dannyroberts/received_events","type":"User","site_admin":false},"body":"The first step here isn't documented, but this now essentially works.\r\n\r\nCreate your spec:\r\n```\r\ncp commcare-cloud-bootstrap/specs/{example,droberts}_spec.yml\r\n# edit your spec to match your own AWS configuration\r\nvim commcare-cloud-bootstrap/specs/droberts_spec.yml\r\n```\r\n\r\nCreate a provision a new environment named `banana` based on the spec:\r\n```\r\ncommcare-cloud-bootstrap privision commcare-cloud-bootstrap/specs/droberts_spec.yml --env banana\r\n```\r\nLook at the EC2 instances associated with this env\r\n```\r\ncommcare-cloud-bootstrap show banana\r\n```\r\n\r\nApply our basic user configuration to the new machines (so you can login with your own private key, as all following ansible steps require):\r\n\r\n```\r\ncommcare-cloud banana bootstrap-users\r\n```\r\nAnd then deploy the whole stack!\r\n\r\n```\r\ncommcare-cloud banana deploy-stack --skip-check --quiet\r\n```\r\nThere are a few places that this script will halt with an error, but if you start it back up where you left off using `--start-at-task` it'll continue without error. Guessing these are just timing issues (waiting for a service to start up, etc.), but I'll be looking into these next to make this process run smoothly from start to finish.\r\n\r\n# restarting EC2 instances\r\n\r\nYou may want to have a throwaway env live for a couple days without actually paying for a couple days; so it'll be common to shut down the EC2 instances over night, for example. If you've restarted your EC2 instances, they'll have new public IPs, which matters because we ssh into them using their public IPs. So after restarting them, you'll need to update the inventory file to have the new public IPs:\r\n\r\n```\r\ncommcare-cloud-bootstrap reip banana\r\n```\r\n\r\nand then you can run the after-reboot command:\r\n```\r\ncommcare-cloud banana after-reboot\r\n```","created_at":"2018-02-13T20:49:59Z","updated_at":"2018-02-14T20:40:22Z","closed_at":null,"merged_at":null,"merge_commit_sha":"2696f1bf1f5634b28dffbd0398b7167b26cae70d","assignee":null,"assignees":[],"requested_reviewers":[{"login":"javierwilson","id":596817,"avatar_url":"https://avatars1.githubusercontent.com/u/596817?v=4","gravatar_id":"","url":"https://api.github.com/users/javierwilson","html_url":"https://github.com/javierwilson","followers_url":"https://api.github.com/users/javierwilson/followers","following_url":"https://api.github.com/users/javierwilson/following{/other_user}","gists_url":"https://api.github.com/users/javierwilson/gists{/gist_id}","starred_url":"https://api.github.com/users/javierwilson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/javierwilson/subscriptions","organizations_url":"https://api.github.com/users/javierwilson/orgs","repos_url":"https://api.github.com/users/javierwilson/repos","events_url":"https://api.github.com/users/javierwilson/events{/privacy}","received_events_url":"https://api.github.com/users/javierwilson/received_events","type":"User","site_admin":false}],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/dimagi/commcare-cloud/pulls/1348/commits","review_comments_url":"https://api.github.com/repos/dimagi/commcare-cloud/pulls/1348/comments","review_comment_url":"https://api.github.com/repos/dimagi/commcare-cloud/pulls/comments{/number}","comments_url":"https://api.github.com/repos/dimagi/commcare-cloud/issues/1348/comments","statuses_url":"https://api.github.com/repos/dimagi/commcare-cloud/statuses/1d1d4157482b3f3171d534895a8e4a7e47ec389a","head":{"label":"dimagi:dmr/commcare-cloud-bootstrap","ref":"dmr/commcare-cloud-bootstrap","sha":"1d1d4157482b3f3171d534895a8e4a7e47ec389a","user":{"login":"dimagi","id":94514,"avatar_url":"https://avatars1.githubusercontent.com/u/94514?v=4","gravatar_id":"","url":"https://api.github.com/users/dimagi","html_url":"https://github.com/dimagi","followers_url":"https://api.github.com/users/dimagi/followers","following_url":"https://api.github.com/users/dimagi/following{/other_user}","gists_url":"https://api.github.com/users/dimagi/gists{/gist_id}","starred_url":"https://api.github.com/users/dimagi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dimagi/subscriptions","organizations_url":"https://api.github.com/users/dimagi/orgs","repos_url":"https://api.github.com/users/dimagi/repos","events_url":"https://api.github.com/users/dimagi/events{/privacy}","received_events_url":"https://api.github.com/users/dimagi/received_events","type":"Organization","site_admin":false},"repo":{"id":22074543,"name":"commcare-cloud","full_name":"dimagi/commcare-cloud","owner":{"login":"dimagi","id":94514,"avatar_url":"https://avatars1.githubusercontent.com/u/94514?v=4","gravatar_id":"","url":"https://api.github.com/users/dimagi","html_url":"https://github.com/dimagi","followers_url":"https://api.github.com/users/dimagi/followers","following_url":"https://api.github.com/users/dimagi/following{/other_user}","gists_url":"https://api.github.com/users/dimagi/gists{/gist_id}","starred_url":"https://api.github.com/users/dimagi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dimagi/subscriptions","organizations_url":"https://api.github.com/users/dimagi/orgs","repos_url":"https://api.github.com/users/dimagi/repos","events_url":"https://api.github.com/users/dimagi/events{/privacy}","received_events_url":"https://api.github.com/users/dimagi/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/dimagi/commcare-cloud","description":null,"fork":false,"url":"https://api.github.com/repos/dimagi/commcare-cloud","forks_url":"https://api.github.com/repos/dimagi/commcare-cloud/forks","keys_url":"https://api.github.com/repos/dimagi/commcare-cloud/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dimagi/commcare-cloud/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dimagi/commcare-cloud/teams","hooks_url":"https://api.github.com/repos/dimagi/commcare-cloud/hooks","issue_events_url":"https://api.github.com/repos/dimagi/commcare-cloud/issues/events{/number}","events_url":"https://api.github.com/repos/dimagi/commcare-cloud/events","assignees_url":"https://api.github.com/repos/dimagi/commcare-cloud/assignees{/user}","branches_url":"https://api.github.com/repos/dimagi/commcare-cloud/branches{/branch}","tags_url":"https://api.github.com/repos/dimagi/commcare-cloud/tags","blobs_url":"https://api.github.com/repos/dimagi/commcare-cloud/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dimagi/commcare-cloud/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dimagi/commcare-cloud/git/refs{/sha}","trees_url":"https://api.github.com/repos/dimagi/commcare-cloud/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dimagi/commcare-cloud/statuses/{sha}","languages_url":"https://api.github.com/repos/dimagi/commcare-cloud/languages","stargazers_url":"https://api.github.com/repos/dimagi/commcare-cloud/stargazers","contributors_url":"https://api.github.com/repos/dimagi/commcare-cloud/contributors","subscribers_url":"https://api.github.com/repos/dimagi/commcare-cloud/subscribers","subscription_url":"https://api.github.com/repos/dimagi/commcare-cloud/subscription","commits_url":"https://api.github.com/repos/dimagi/commcare-cloud/commits{/sha}","git_commits_url":"https://api.github.com/repos/dimagi/commcare-cloud/git/commits{/sha}","comments_url":"https://api.github.com/repos/dimagi/commcare-cloud/comments{/number}","issue_comment_url":"https://api.github.com/repos/dimagi/commcare-cloud/issues/comments{/number}","contents_url":"https://api.github.com/repos/dimagi/commcare-cloud/contents/{+path}","compare_url":"https://api.github.com/repos/dimagi/commcare-cloud/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dimagi/commcare-cloud/merges","archive_url":"https://api.github.com/repos/dimagi/commcare-cloud/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dimagi/commcare-cloud/downloads","issues_url":"https://api.github.com/repos/dimagi/commcare-cloud/issues{/number}","pulls_url":"https://api.github.com/repos/dimagi/commcare-cloud/pulls{/number}","milestones_url":"https://api.github.com/repos/dimagi/commcare-cloud/milestones{/number}","notifications_url":"https://api.github.com/repos/dimagi/commcare-cloud/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dimagi/commcare-cloud/labels{/name}","releases_url":"https://api.github.com/repos/dimagi/commcare-cloud/releases{/id}","deployments_url":"https://api.github.com/repos/dimagi/commcare-cloud/deployments","created_at":"2014-07-21T18:04:26Z","updated_at":"2018-01-16T22:35:22Z","pushed_at":"2018-02-14T20:16:37Z","git_url":"git://github.com/dimagi/commcare-cloud.git","ssh_url":"git@github.com:dimagi/commcare-cloud.git","clone_url":"https://github.com/dimagi/commcare-cloud.git","svn_url":"https://github.com/dimagi/commcare-cloud","homepage":null,"size":6061,"stargazers_count":7,"watchers_count":7,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":8,"mirror_url":null,"archived":false,"open_issues_count":12,"license":null,"forks":8,"open_issues":12,"watchers":7,"default_branch":"master"}},"base":{"label":"dimagi:master","ref":"master","sha":"4e393fbfbbb1387c801e40e4a29afb9cfe979fcc","user":{"login":"dimagi","id":94514,"avatar_url":"https://avatars1.githubusercontent.com/u/94514?v=4","gravatar_id":"","url":"https://api.github.com/users/dimagi","html_url":"https://github.com/dimagi","followers_url":"https://api.github.com/users/dimagi/followers","following_url":"https://api.github.com/users/dimagi/following{/other_user}","gists_url":"https://api.github.com/users/dimagi/gists{/gist_id}","starred_url":"https://api.github.com/users/dimagi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dimagi/subscriptions","organizations_url":"https://api.github.com/users/dimagi/orgs","repos_url":"https://api.github.com/users/dimagi/repos","events_url":"https://api.github.com/users/dimagi/events{/privacy}","received_events_url":"https://api.github.com/users/dimagi/received_events","type":"Organization","site_admin":false},"repo":{"id":22074543,"name":"commcare-cloud","full_name":"dimagi/commcare-cloud","owner":{"login":"dimagi","id":94514,"avatar_url":"https://avatars1.githubusercontent.com/u/94514?v=4","gravatar_id":"","url":"https://api.github.com/users/dimagi","html_url":"https://github.com/dimagi","followers_url":"https://api.github.com/users/dimagi/followers","following_url":"https://api.github.com/users/dimagi/following{/other_user}","gists_url":"https://api.github.com/users/dimagi/gists{/gist_id}","starred_url":"https://api.github.com/users/dimagi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dimagi/subscriptions","organizations_url":"https://api.github.com/users/dimagi/orgs","repos_url":"https://api.github.com/users/dimagi/repos","events_url":"https://api.github.com/users/dimagi/events{/privacy}","received_events_url":"https://api.github.com/users/dimagi/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/dimagi/commcare-cloud","description":null,"fork":false,"url":"https://api.github.com/repos/dimagi/commcare-cloud","forks_url":"https://api.github.com/repos/dimagi/commcare-cloud/forks","keys_url":"https://api.github.com/repos/dimagi/commcare-cloud/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dimagi/commcare-cloud/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dimagi/commcare-cloud/teams","hooks_url":"https://api.github.com/repos/dimagi/commcare-cloud/hooks","issue_events_url":"https://api.github.com/repos/dimagi/commcare-cloud/issues/events{/number}","events_url":"https://api.github.com/repos/dimagi/commcare-cloud/events","assignees_url":"https://api.github.com/repos/dimagi/commcare-cloud/assignees{/user}","branches_url":"https://api.github.com/repos/dimagi/commcare-cloud/branches{/branch}","tags_url":"https://api.github.com/repos/dimagi/commcare-cloud/tags","blobs_url":"https://api.github.com/repos/dimagi/commcare-cloud/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dimagi/commcare-cloud/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dimagi/commcare-cloud/git/refs{/sha}","trees_url":"https://api.github.com/repos/dimagi/commcare-cloud/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dimagi/commcare-cloud/statuses/{sha}","languages_url":"https://api.github.com/repos/dimagi/commcare-cloud/languages","stargazers_url":"https://api.github.com/repos/dimagi/commcare-cloud/stargazers","contributors_url":"https://api.github.com/repos/dimagi/commcare-cloud/contributors","subscribers_url":"https://api.github.com/repos/dimagi/commcare-cloud/subscribers","subscription_url":"https://api.github.com/repos/dimagi/commcare-cloud/subscription","commits_url":"https://api.github.com/repos/dimagi/commcare-cloud/commits{/sha}","git_commits_url":"https://api.github.com/repos/dimagi/commcare-cloud/git/commits{/sha}","comments_url":"https://api.github.com/repos/dimagi/commcare-cloud/comments{/number}","issue_comment_url":"https://api.github.com/repos/dimagi/commcare-cloud/issues/comments{/number}","contents_url":"https://api.github.com/repos/dimagi/commcare-cloud/contents/{+path}","compare_url":"https://api.github.com/repos/dimagi/commcare-cloud/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dimagi/commcare-cloud/merges","archive_url":"https://api.github.com/repos/dimagi/commcare-cloud/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dimagi/commcare-cloud/downloads","issues_url":"https://api.github.com/repos/dimagi/commcare-cloud/issues{/number}","pulls_url":"https://api.github.com/repos/dimagi/commcare-cloud/pulls{/number}","milestones_url":"https://api.github.com/repos/dimagi/commcare-cloud/milestones{/number}","notifications_url":"https://api.github.com/repos/dimagi/commcare-cloud/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dimagi/commcare-cloud/labels{/name}","releases_url":"https://api.github.com/repos/dimagi/commcare-cloud/releases{/id}","deployments_url":"https://api.github.com/repos/dimagi/commcare-cloud/deployments","created_at":"2014-07-21T18:04:26Z","updated_at":"2018-01-16T22:35:22Z","pushed_at":"2018-02-14T20:16:37Z","git_url":"git://github.com/dimagi/commcare-cloud.git","ssh_url":"git@github.com:dimagi/commcare-cloud.git","clone_url":"https://github.com/dimagi/commcare-cloud.git","svn_url":"https://github.com/dimagi/commcare-cloud","homepage":null,"size":6061,"stargazers_count":7,"watchers_count":7,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":8,"mirror_url":null,"archived":false,"open_issues_count":12,"license":null,"forks":8,"open_issues":12,"watchers":7,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/dimagi/commcare-cloud/pulls/1348"},"html":{"href":"https://github.com/dimagi/commcare-cloud/pull/1348"},"issue":{"href":"https://api.github.com/repos/dimagi/commcare-cloud/issues/1348"},"comments":{"href":"https://api.github.com/repos/dimagi/commcare-cloud/issues/1348/comments"},"review_comments":{"href":"https://api.github.com/repos/dimagi/commcare-cloud/pulls/1348/comments"},"review_comment":{"href":"https://api.github.com/repos/dimagi/commcare-cloud/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/dimagi/commcare-cloud/pulls/1348/commits"},"statuses":{"href":"https://api.github.com/repos/dimagi/commcare-cloud/statuses/1d1d4157482b3f3171d534895a8e4a7e47ec389a"}},"author_association":"OWNER"}}
{"actor":{"display_login":"dannyroberts"}}