{"action":"created","comment":{"url":"https://api.github.com/repos/DataDog/integrations-core/pulls/comments/180047239","pull_request_review_id":110397899,"id":180047239,"diff_hunk":"@@ -844,24 +850,60 @@ def get_stats_for_single_hypervisor(self, hyp_id, instance, host_tags=None):\n             for i, avg in enumerate([1, 5, 15]):\n                 self.gauge('openstack.nova.hypervisor_load.{0}'.format(avg), load_averages[i], tags=tags)\n \n-    def get_all_server_ids(self, filter_by_host=None):\n+    # Get all of the server IDs and their metadata and cache them\n+    # After the first run, we will only get servers that have changed state since the last collection run\n+    def get_all_servers(self, i_key, filter_by_host=None):\n         query_params = {}\n         if filter_by_host:\n             query_params[\"host\"] = filter_by_host\n \n-        url = '{0}/servers'.format(self.get_nova_endpoint())\n+        # If we don't have a timestamp for this instance, default to None\n+        if i_key not in self.changes_since_time:\n+            self._changes_since_time[i_key] = None\n+\n+        if self.changes_since_time[i_key]:\n+            query_params['changes-since'] = changes_since_time","path":"openstack/datadog_checks/openstack/openstack.py","position":60,"original_position":31,"commit_id":"0662257c43d06e3b11bfc7ce77d1fdaa1c054c1c","original_commit_id":"fa3dc7d0c5e6809dcf32a76b6b672d6b60db6f4f","user":{"login":"truthbk","id":448216,"avatar_url":"https://avatars0.githubusercontent.com/u/448216?v=4","gravatar_id":"","url":"https://api.github.com/users/truthbk","html_url":"https://github.com/truthbk","followers_url":"https://api.github.com/users/truthbk/followers","following_url":"https://api.github.com/users/truthbk/following{/other_user}","gists_url":"https://api.github.com/users/truthbk/gists{/gist_id}","starred_url":"https://api.github.com/users/truthbk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/truthbk/subscriptions","organizations_url":"https://api.github.com/users/truthbk/orgs","repos_url":"https://api.github.com/users/truthbk/repos","events_url":"https://api.github.com/users/truthbk/events{/privacy}","received_events_url":"https://api.github.com/users/truthbk/received_events","type":"User","site_admin":false},"body":"You could use a defaultdict, or `self.changes_since_time.get(i_key, None)`. ","created_at":"2018-04-09T10:12:02Z","updated_at":"2018-04-09T10:57:53Z","html_url":"https://github.com/DataDog/integrations-core/pull/1352#discussion_r180047239","pull_request_url":"https://api.github.com/repos/DataDog/integrations-core/pulls/1352","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/DataDog/integrations-core/pulls/comments/180047239"},"html":{"href":"https://github.com/DataDog/integrations-core/pull/1352#discussion_r180047239"},"pull_request":{"href":"https://api.github.com/repos/DataDog/integrations-core/pulls/1352"}},"in_reply_to_id":179864722},"pull_request":{"url":"https://api.github.com/repos/DataDog/integrations-core/pulls/1352","id":179737305,"html_url":"https://github.com/DataDog/integrations-core/pull/1352","diff_url":"https://github.com/DataDog/integrations-core/pull/1352.diff","patch_url":"https://github.com/DataDog/integrations-core/pull/1352.patch","issue_url":"https://api.github.com/repos/DataDog/integrations-core/issues/1352","number":1352,"state":"open","locked":false,"title":"[openstack] Nova API call optimizations","user":{"login":"nmuesch","id":8941884,"avatar_url":"https://avatars0.githubusercontent.com/u/8941884?v=4","gravatar_id":"","url":"https://api.github.com/users/nmuesch","html_url":"https://github.com/nmuesch","followers_url":"https://api.github.com/users/nmuesch/followers","following_url":"https://api.github.com/users/nmuesch/following{/other_user}","gists_url":"https://api.github.com/users/nmuesch/gists{/gist_id}","starred_url":"https://api.github.com/users/nmuesch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nmuesch/subscriptions","organizations_url":"https://api.github.com/users/nmuesch/orgs","repos_url":"https://api.github.com/users/nmuesch/repos","events_url":"https://api.github.com/users/nmuesch/events{/privacy}","received_events_url":"https://api.github.com/users/nmuesch/received_events","type":"User","site_admin":false},"body":"<!-- *Note: Please remember to review the Datadog [Contribution Guidelines](https://github.com/DataDog/dd-agent/blob/master/CONTRIBUTING.md)\r\nif you have not yet done so.* -->\r\n\r\n### What does this PR do?\r\n\r\nUses the optional `changes-since` portion of the Nova API to not query a full list of things every time we hit the `/servers`/ endpoint. \r\n\r\nAdditionally, we currently call the `servers` endpoint to get a list of all servers, then check `servers/{id}` to get some metadata about each individual server, then call the diagnostics endpoint. on that server if the server is in an active state.\r\n\r\nInstead, we now call the `servers/detail` endpoint on the first check run, which will cache all metadata and server ids. We let nova filter this to only get servers in the `ACTIVE` state, to reduce payload sizes. Then we call the diagnostic method on all active servers. This removes the call to `server/{id}` entirely (which is called once for each server running)\r\n\r\nOn top of that, now we cache the list of servers, so that heavy call to `servers/details` only occurs on the first check run. Then we keep a timestamp of when we last called it, and on subsequent calls/check runs only get the list of servers that have had their state changed. \r\n\r\n### Motivation\r\n\r\nWe are making heavy calls to the Nova API, and a lot of them. Which causes the check to run long, and put a burden on the Openstack API when there are a large number of hosts. \r\n\r\n### Testing Guidelines\r\n\r\nAn overview on [testing](https://github.com/DataDog/dd-agent/blob/master/tests/README.md)\r\nis available in our contribution guidelines.\r\n\r\n### Versioning\r\n\r\n- [ ] Bumped the check version in `manifest.json`\r\n- [ ] Bumped the check version in `datadog_checks/{integration}/__init__.py`\r\n- [ ] Updated `CHANGELOG.md`. Please use `Unreleased` as the date in the title\r\n  for the new section.\r\n- [ ] If PR impacts documentation, docs team has been notified or [an issue has been opened on the documentation repo](https://github.com/DataDog/documentation/issues/new) \r\n\r\n### Additional Notes\r\n\r\nAnything else we should know when reviewing?\r\n","created_at":"2018-04-05T16:57:23Z","updated_at":"2018-04-09T10:57:54Z","closed_at":null,"merged_at":null,"merge_commit_sha":"c505567f9c6987a9221f9e97e86ade1307952ea4","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[{"id":581291045,"url":"https://api.github.com/repos/DataDog/integrations-core/labels/do-not-merge/WIP","name":"do-not-merge/WIP","color":"000000","default":false}],"milestone":null,"commits_url":"https://api.github.com/repos/DataDog/integrations-core/pulls/1352/commits","review_comments_url":"https://api.github.com/repos/DataDog/integrations-core/pulls/1352/comments","review_comment_url":"https://api.github.com/repos/DataDog/integrations-core/pulls/comments{/number}","comments_url":"https://api.github.com/repos/DataDog/integrations-core/issues/1352/comments","statuses_url":"https://api.github.com/repos/DataDog/integrations-core/statuses/0662257c43d06e3b11bfc7ce77d1fdaa1c054c1c","head":{"label":"DataDog:nick/cache_server_list","ref":"nick/cache_server_list","sha":"0662257c43d06e3b11bfc7ce77d1fdaa1c054c1c","user":{"login":"DataDog","id":365230,"avatar_url":"https://avatars1.githubusercontent.com/u/365230?v=4","gravatar_id":"","url":"https://api.github.com/users/DataDog","html_url":"https://github.com/DataDog","followers_url":"https://api.github.com/users/DataDog/followers","following_url":"https://api.github.com/users/DataDog/following{/other_user}","gists_url":"https://api.github.com/users/DataDog/gists{/gist_id}","starred_url":"https://api.github.com/users/DataDog/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DataDog/subscriptions","organizations_url":"https://api.github.com/users/DataDog/orgs","repos_url":"https://api.github.com/users/DataDog/repos","events_url":"https://api.github.com/users/DataDog/events{/privacy}","received_events_url":"https://api.github.com/users/DataDog/received_events","type":"Organization","site_admin":false},"repo":{"id":47203045,"name":"integrations-core","full_name":"DataDog/integrations-core","owner":{"login":"DataDog","id":365230,"avatar_url":"https://avatars1.githubusercontent.com/u/365230?v=4","gravatar_id":"","url":"https://api.github.com/users/DataDog","html_url":"https://github.com/DataDog","followers_url":"https://api.github.com/users/DataDog/followers","following_url":"https://api.github.com/users/DataDog/following{/other_user}","gists_url":"https://api.github.com/users/DataDog/gists{/gist_id}","starred_url":"https://api.github.com/users/DataDog/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DataDog/subscriptions","organizations_url":"https://api.github.com/users/DataDog/orgs","repos_url":"https://api.github.com/users/DataDog/repos","events_url":"https://api.github.com/users/DataDog/events{/privacy}","received_events_url":"https://api.github.com/users/DataDog/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/DataDog/integrations-core","description":"Core integrations of the datadog agent","fork":false,"url":"https://api.github.com/repos/DataDog/integrations-core","forks_url":"https://api.github.com/repos/DataDog/integrations-core/forks","keys_url":"https://api.github.com/repos/DataDog/integrations-core/keys{/key_id}","collaborators_url":"https://api.github.com/repos/DataDog/integrations-core/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/DataDog/integrations-core/teams","hooks_url":"https://api.github.com/repos/DataDog/integrations-core/hooks","issue_events_url":"https://api.github.com/repos/DataDog/integrations-core/issues/events{/number}","events_url":"https://api.github.com/repos/DataDog/integrations-core/events","assignees_url":"https://api.github.com/repos/DataDog/integrations-core/assignees{/user}","branches_url":"https://api.github.com/repos/DataDog/integrations-core/branches{/branch}","tags_url":"https://api.github.com/repos/DataDog/integrations-core/tags","blobs_url":"https://api.github.com/repos/DataDog/integrations-core/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/DataDog/integrations-core/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/DataDog/integrations-core/git/refs{/sha}","trees_url":"https://api.github.com/repos/DataDog/integrations-core/git/trees{/sha}","statuses_url":"https://api.github.com/repos/DataDog/integrations-core/statuses/{sha}","languages_url":"https://api.github.com/repos/DataDog/integrations-core/languages","stargazers_url":"https://api.github.com/repos/DataDog/integrations-core/stargazers","contributors_url":"https://api.github.com/repos/DataDog/integrations-core/contributors","subscribers_url":"https://api.github.com/repos/DataDog/integrations-core/subscribers","subscription_url":"https://api.github.com/repos/DataDog/integrations-core/subscription","commits_url":"https://api.github.com/repos/DataDog/integrations-core/commits{/sha}","git_commits_url":"https://api.github.com/repos/DataDog/integrations-core/git/commits{/sha}","comments_url":"https://api.github.com/repos/DataDog/integrations-core/comments{/number}","issue_comment_url":"https://api.github.com/repos/DataDog/integrations-core/issues/comments{/number}","contents_url":"https://api.github.com/repos/DataDog/integrations-core/contents/{+path}","compare_url":"https://api.github.com/repos/DataDog/integrations-core/compare/{base}...{head}","merges_url":"https://api.github.com/repos/DataDog/integrations-core/merges","archive_url":"https://api.github.com/repos/DataDog/integrations-core/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/DataDog/integrations-core/downloads","issues_url":"https://api.github.com/repos/DataDog/integrations-core/issues{/number}","pulls_url":"https://api.github.com/repos/DataDog/integrations-core/pulls{/number}","milestones_url":"https://api.github.com/repos/DataDog/integrations-core/milestones{/number}","notifications_url":"https://api.github.com/repos/DataDog/integrations-core/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/DataDog/integrations-core/labels{/name}","releases_url":"https://api.github.com/repos/DataDog/integrations-core/releases{/id}","deployments_url":"https://api.github.com/repos/DataDog/integrations-core/deployments","created_at":"2015-12-01T16:41:45Z","updated_at":"2018-04-09T10:45:21Z","pushed_at":"2018-04-09T10:45:15Z","git_url":"git://github.com/DataDog/integrations-core.git","ssh_url":"git@github.com:DataDog/integrations-core.git","clone_url":"https://github.com/DataDog/integrations-core.git","svn_url":"https://github.com/DataDog/integrations-core","homepage":null,"size":8472,"stargazers_count":99,"watchers_count":99,"language":"Python","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":221,"mirror_url":null,"archived":false,"open_issues_count":81,"license":{"key":"bsd-3-clause","name":"BSD 3-Clause \"New\" or \"Revised\" License","spdx_id":"BSD-3-Clause","url":"https://api.github.com/licenses/bsd-3-clause"},"forks":221,"open_issues":81,"watchers":99,"default_branch":"master"}},"base":{"label":"DataDog:jaime/openstack_unscoped","ref":"jaime/openstack_unscoped","sha":"5406ba7646017e777958d7d97dc128a9ee831b0f","user":{"login":"DataDog","id":365230,"avatar_url":"https://avatars1.githubusercontent.com/u/365230?v=4","gravatar_id":"","url":"https://api.github.com/users/DataDog","html_url":"https://github.com/DataDog","followers_url":"https://api.github.com/users/DataDog/followers","following_url":"https://api.github.com/users/DataDog/following{/other_user}","gists_url":"https://api.github.com/users/DataDog/gists{/gist_id}","starred_url":"https://api.github.com/users/DataDog/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DataDog/subscriptions","organizations_url":"https://api.github.com/users/DataDog/orgs","repos_url":"https://api.github.com/users/DataDog/repos","events_url":"https://api.github.com/users/DataDog/events{/privacy}","received_events_url":"https://api.github.com/users/DataDog/received_events","type":"Organization","site_admin":false},"repo":{"id":47203045,"name":"integrations-core","full_name":"DataDog/integrations-core","owner":{"login":"DataDog","id":365230,"avatar_url":"https://avatars1.githubusercontent.com/u/365230?v=4","gravatar_id":"","url":"https://api.github.com/users/DataDog","html_url":"https://github.com/DataDog","followers_url":"https://api.github.com/users/DataDog/followers","following_url":"https://api.github.com/users/DataDog/following{/other_user}","gists_url":"https://api.github.com/users/DataDog/gists{/gist_id}","starred_url":"https://api.github.com/users/DataDog/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DataDog/subscriptions","organizations_url":"https://api.github.com/users/DataDog/orgs","repos_url":"https://api.github.com/users/DataDog/repos","events_url":"https://api.github.com/users/DataDog/events{/privacy}","received_events_url":"https://api.github.com/users/DataDog/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/DataDog/integrations-core","description":"Core integrations of the datadog agent","fork":false,"url":"https://api.github.com/repos/DataDog/integrations-core","forks_url":"https://api.github.com/repos/DataDog/integrations-core/forks","keys_url":"https://api.github.com/repos/DataDog/integrations-core/keys{/key_id}","collaborators_url":"https://api.github.com/repos/DataDog/integrations-core/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/DataDog/integrations-core/teams","hooks_url":"https://api.github.com/repos/DataDog/integrations-core/hooks","issue_events_url":"https://api.github.com/repos/DataDog/integrations-core/issues/events{/number}","events_url":"https://api.github.com/repos/DataDog/integrations-core/events","assignees_url":"https://api.github.com/repos/DataDog/integrations-core/assignees{/user}","branches_url":"https://api.github.com/repos/DataDog/integrations-core/branches{/branch}","tags_url":"https://api.github.com/repos/DataDog/integrations-core/tags","blobs_url":"https://api.github.com/repos/DataDog/integrations-core/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/DataDog/integrations-core/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/DataDog/integrations-core/git/refs{/sha}","trees_url":"https://api.github.com/repos/DataDog/integrations-core/git/trees{/sha}","statuses_url":"https://api.github.com/repos/DataDog/integrations-core/statuses/{sha}","languages_url":"https://api.github.com/repos/DataDog/integrations-core/languages","stargazers_url":"https://api.github.com/repos/DataDog/integrations-core/stargazers","contributors_url":"https://api.github.com/repos/DataDog/integrations-core/contributors","subscribers_url":"https://api.github.com/repos/DataDog/integrations-core/subscribers","subscription_url":"https://api.github.com/repos/DataDog/integrations-core/subscription","commits_url":"https://api.github.com/repos/DataDog/integrations-core/commits{/sha}","git_commits_url":"https://api.github.com/repos/DataDog/integrations-core/git/commits{/sha}","comments_url":"https://api.github.com/repos/DataDog/integrations-core/comments{/number}","issue_comment_url":"https://api.github.com/repos/DataDog/integrations-core/issues/comments{/number}","contents_url":"https://api.github.com/repos/DataDog/integrations-core/contents/{+path}","compare_url":"https://api.github.com/repos/DataDog/integrations-core/compare/{base}...{head}","merges_url":"https://api.github.com/repos/DataDog/integrations-core/merges","archive_url":"https://api.github.com/repos/DataDog/integrations-core/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/DataDog/integrations-core/downloads","issues_url":"https://api.github.com/repos/DataDog/integrations-core/issues{/number}","pulls_url":"https://api.github.com/repos/DataDog/integrations-core/pulls{/number}","milestones_url":"https://api.github.com/repos/DataDog/integrations-core/milestones{/number}","notifications_url":"https://api.github.com/repos/DataDog/integrations-core/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/DataDog/integrations-core/labels{/name}","releases_url":"https://api.github.com/repos/DataDog/integrations-core/releases{/id}","deployments_url":"https://api.github.com/repos/DataDog/integrations-core/deployments","created_at":"2015-12-01T16:41:45Z","updated_at":"2018-04-09T10:45:21Z","pushed_at":"2018-04-09T10:45:15Z","git_url":"git://github.com/DataDog/integrations-core.git","ssh_url":"git@github.com:DataDog/integrations-core.git","clone_url":"https://github.com/DataDog/integrations-core.git","svn_url":"https://github.com/DataDog/integrations-core","homepage":null,"size":8472,"stargazers_count":99,"watchers_count":99,"language":"Python","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":221,"mirror_url":null,"archived":false,"open_issues_count":81,"license":{"key":"bsd-3-clause","name":"BSD 3-Clause \"New\" or \"Revised\" License","spdx_id":"BSD-3-Clause","url":"https://api.github.com/licenses/bsd-3-clause"},"forks":221,"open_issues":81,"watchers":99,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/DataDog/integrations-core/pulls/1352"},"html":{"href":"https://github.com/DataDog/integrations-core/pull/1352"},"issue":{"href":"https://api.github.com/repos/DataDog/integrations-core/issues/1352"},"comments":{"href":"https://api.github.com/repos/DataDog/integrations-core/issues/1352/comments"},"review_comments":{"href":"https://api.github.com/repos/DataDog/integrations-core/pulls/1352/comments"},"review_comment":{"href":"https://api.github.com/repos/DataDog/integrations-core/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/DataDog/integrations-core/pulls/1352/commits"},"statuses":{"href":"https://api.github.com/repos/DataDog/integrations-core/statuses/0662257c43d06e3b11bfc7ce77d1fdaa1c054c1c"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"truthbk"}}