{"action":"created","comment":{"url":"https://api.github.com/repos/DataDog/integrations-core/pulls/comments/216064625","pull_request_review_id":153474152,"id":216064625,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIxNjA2NDYyNQ==","diff_hunk":"@@ -945,9 +919,37 @@ def get_all_servers(self, i_key, collect_all_tenants, filter_by_host=None):\n                 except KeyError as e:\n                     self.log.debug(\"Server: %s has already been removed from the cache\", new_server['server_id'])\n \n-        return self.server_details_by_id\n+    def filter_excluded_servers(self):\n+        proj_list = set([])\n+        servers_copy = copy.deepcopy(self.server_details_by_id)\n+        if self.exclude_server_id_rules:\n+            # Filter out excluded servers\n+            for exclude_id_rule in self.exclude_server_id_rules:\n+                for server_id in self.server_details_by_id.keys():\n+                    if re.match(exclude_id_rule, server_id):\n+                        del self.server_details_by_id[server_id]\n+\n+        for server in self.server_details_by_id.iteritems():\n+            proj_list.add(server[1].get('project_name'))\n+\n+        projects_filtered = pattern_filter(proj_list,\n+                                           whitelist=self.include_project_name_rules,\n+                                           blacklist=self.exclude_project_name_rules\n+                                           )\n+        for server in servers_copy.iteritems():\n+            if server[1].get('project_name') not in projects_filtered:\n+                # This may have already been deleted if the server belongs to both\n+                # the project blacklist AND the excluded server id list\n+                try:\n+                    del self.server_details_by_id[server[0]]\n+                except KeyError:\n+                    self.log.debug(\"Server %s was already removed from the list\", server[0])\n \n     def get_project_name_from_id(self, tenant_id):\n+        # [TODO]","path":"openstack/datadog_checks/openstack/openstack.py","position":216,"original_position":216,"commit_id":"96cb93de3ae49f06e8cd3f78223e3a8d378d8825","original_commit_id":"96cb93de3ae49f06e8cd3f78223e3a8d378d8825","user":{"login":"nmuesch","id":8941884,"node_id":"MDQ6VXNlcjg5NDE4ODQ=","avatar_url":"https://avatars0.githubusercontent.com/u/8941884?v=4","gravatar_id":"","url":"https://api.github.com/users/nmuesch","html_url":"https://github.com/nmuesch","followers_url":"https://api.github.com/users/nmuesch/followers","following_url":"https://api.github.com/users/nmuesch/following{/other_user}","gists_url":"https://api.github.com/users/nmuesch/gists{/gist_id}","starred_url":"https://api.github.com/users/nmuesch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nmuesch/subscriptions","organizations_url":"https://api.github.com/users/nmuesch/orgs","repos_url":"https://api.github.com/users/nmuesch/repos","events_url":"https://api.github.com/users/nmuesch/events{/privacy}","received_events_url":"https://api.github.com/users/nmuesch/received_events","type":"User","site_admin":false},"body":"Will do 👍 Just wanted to make sure we kept track of it somewhere. ","created_at":"2018-09-07T19:28:08Z","updated_at":"2018-09-07T19:28:08Z","html_url":"https://github.com/DataDog/integrations-core/pull/2161#discussion_r216064625","pull_request_url":"https://api.github.com/repos/DataDog/integrations-core/pulls/2161","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/DataDog/integrations-core/pulls/comments/216064625"},"html":{"href":"https://github.com/DataDog/integrations-core/pull/2161#discussion_r216064625"},"pull_request":{"href":"https://api.github.com/repos/DataDog/integrations-core/pulls/2161"}},"in_reply_to_id":216011271},"pull_request":{"url":"https://api.github.com/repos/DataDog/integrations-core/pulls/2161","id":212797711,"node_id":"MDExOlB1bGxSZXF1ZXN0MjEyNzk3NzEx","html_url":"https://github.com/DataDog/integrations-core/pull/2161","diff_url":"https://github.com/DataDog/integrations-core/pull/2161.diff","patch_url":"https://github.com/DataDog/integrations-core/pull/2161.patch","issue_url":"https://api.github.com/repos/DataDog/integrations-core/issues/2161","number":2161,"state":"open","locked":false,"title":"OpenStack Change check from per hypervisor to control plane","user":{"login":"gzussa","id":280942,"node_id":"MDQ6VXNlcjI4MDk0Mg==","avatar_url":"https://avatars3.githubusercontent.com/u/280942?v=4","gravatar_id":"","url":"https://api.github.com/users/gzussa","html_url":"https://github.com/gzussa","followers_url":"https://api.github.com/users/gzussa/followers","following_url":"https://api.github.com/users/gzussa/following{/other_user}","gists_url":"https://api.github.com/users/gzussa/gists{/gist_id}","starred_url":"https://api.github.com/users/gzussa/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gzussa/subscriptions","organizations_url":"https://api.github.com/users/gzussa/orgs","repos_url":"https://api.github.com/users/gzussa/repos","events_url":"https://api.github.com/users/gzussa/events{/privacy}","received_events_url":"https://api.github.com/users/gzussa/received_events","type":"User","site_admin":false},"body":"### What does this PR do?\r\n\r\nThis PR is a refactoring of the OpenStack integration for performance reasons as well as some fixes.\r\nThe main idea here is deploy the integration next to the control panel instead of each hypervisor.\r\n\r\n### Architecture\r\nThe check used to run on each individual hypervisor and collected PROJECT_LIMIT, HYPERVISOR, and SERVER metrics (and service checks about each neutron network setup) \r\n\r\nThis version of the check modifies the design approach to instead have the check run on ONLY the control plane (or another host that has access to the Keystone API URL). More information about the `why` in the Motivation section. \r\n\r\n### Authentication and Scope\r\nThe check now requires an `admin read only` set of permissions, the extent to which will be updated in the README prior to release. The main thing here is that instead of access to a specific project, the `datadog_user` defined for this integration now needs the ability to list all projects, list all servers, list all hypervisors, etc. We DO NOT require any create/delete permissions as the check strictly monitors the environment. \r\n\r\nThe scope here is not only used to get a list of projects IF `collect_all_projects` isn't enabled. So a user in Openstack can be added to a set of projects, if the datadog user is added to those, we will collect limits metrics about those projects, otherwise we can collect :allthethings:\r\n\r\n### Blacklist/Whitelist\r\nThe check now supports the ability to provide a whitelist/blacklist on project_name to filter the collection of metrics. This currently will include the project limits metrics and the server metrics. Restriction of these will improve performance by reducing the number of `requests` being made. \r\n\r\n### Bugfixes\r\nThe Keystone Server Service check was sending a different tag key for whether it was UP/DOWN, which could cause issues with setting up monitors. We now consistently use `keystone_server` as the tag key. \r\n\r\n### Motivation\r\n\r\nThe other hypervisor based checks we have (read: VSphere) run on some node that only needs to talk to the control plane node. While still useful to have the Agent deployed on each hypervisor, its not strictly necessary to have the check run on each hypervisor. This is because there are a sets of metrics that occupy a more global rationale, like project limits, these don't care about the hypervisor level as many hypervisors can belong to a project. \r\n\r\nAdditionally, by authenticating once per check and one call per project to get limits metrics run for ALL hypervisors, we aren't hitting keystone as much and have a reduced load there. \r\n\r\nThe overall number of queries to the database backend and any load balancer in front of the API servers is lower as well since we can make batch requests to get info about all hypervisors/servers instead of making individual calls on each hypervisor. \r\n\r\n### Review checklist\r\n\r\n- [ ] PR has a [meaningful title](https://github.com/DataDog/integrations-core/blob/master/CONTRIBUTING.md#pull-request-title) or PR has the `no-changelog` label attached\r\n- [ ] Feature or bugfix has tests\r\n- [ ] Git history is clean\r\n- [ ] If PR impacts documentation, docs team has been notified or an issue has been opened on the [documentation repo](https://github.com/DataDog/documentation/issues/new)\r\n\r\n### Additional Notes\r\n\r\nThere are some breaking changes in here, and configuration files and how the check is deployed will need to be changed when upgrading to this version of the Integration. The README will be updated to reflect this prior to release. \r\n\r\nThe check also needs to be updated to refactor the backoff logic to only take affect for non batched endpoints. I.e. we should only experience a backoff if we cant get a list or servers or a list of hypervisors, we shouldn't backoff if we can't reach a single server/hypervisor. This should also come prior to the next release. ","created_at":"2018-09-03T17:24:19Z","updated_at":"2018-09-07T19:28:08Z","closed_at":null,"merged_at":null,"merge_commit_sha":"b438a059905addecee01322cad2215732644b1d6","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[{"name":"agent-integrations","id":2704472,"node_id":"MDQ6VGVhbTI3MDQ0NzI=","slug":"agent-integrations","description":"Tiramisu and stuff","privacy":"closed","url":"https://api.github.com/teams/2704472","members_url":"https://api.github.com/teams/2704472/members{/member}","repositories_url":"https://api.github.com/teams/2704472/repos","permission":"pull"}],"labels":[{"id":937215978,"node_id":"MDU6TGFiZWw5MzcyMTU5Nzg=","url":"https://api.github.com/repos/DataDog/integrations-core/labels/changelog/Changed","name":"changelog/Changed","color":"f2983e","default":false},{"id":581291045,"node_id":"MDU6TGFiZWw1ODEyOTEwNDU=","url":"https://api.github.com/repos/DataDog/integrations-core/labels/do-not-merge/WIP","name":"do-not-merge/WIP","color":"000000","default":false},{"id":936575040,"node_id":"MDU6TGFiZWw5MzY1NzUwNDA=","url":"https://api.github.com/repos/DataDog/integrations-core/labels/integration/openstack","name":"integration/openstack","color":"bfdadc","default":false}],"milestone":null,"commits_url":"https://api.github.com/repos/DataDog/integrations-core/pulls/2161/commits","review_comments_url":"https://api.github.com/repos/DataDog/integrations-core/pulls/2161/comments","review_comment_url":"https://api.github.com/repos/DataDog/integrations-core/pulls/comments{/number}","comments_url":"https://api.github.com/repos/DataDog/integrations-core/issues/2161/comments","statuses_url":"https://api.github.com/repos/DataDog/integrations-core/statuses/96cb93de3ae49f06e8cd3f78223e3a8d378d8825","head":{"label":"DataDog:nick/openstack_one_node","ref":"nick/openstack_one_node","sha":"96cb93de3ae49f06e8cd3f78223e3a8d378d8825","user":{"login":"DataDog","id":365230,"node_id":"MDEyOk9yZ2FuaXphdGlvbjM2NTIzMA==","avatar_url":"https://avatars1.githubusercontent.com/u/365230?v=4","gravatar_id":"","url":"https://api.github.com/users/DataDog","html_url":"https://github.com/DataDog","followers_url":"https://api.github.com/users/DataDog/followers","following_url":"https://api.github.com/users/DataDog/following{/other_user}","gists_url":"https://api.github.com/users/DataDog/gists{/gist_id}","starred_url":"https://api.github.com/users/DataDog/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DataDog/subscriptions","organizations_url":"https://api.github.com/users/DataDog/orgs","repos_url":"https://api.github.com/users/DataDog/repos","events_url":"https://api.github.com/users/DataDog/events{/privacy}","received_events_url":"https://api.github.com/users/DataDog/received_events","type":"Organization","site_admin":false},"repo":{"id":47203045,"node_id":"MDEwOlJlcG9zaXRvcnk0NzIwMzA0NQ==","name":"integrations-core","full_name":"DataDog/integrations-core","owner":{"login":"DataDog","id":365230,"node_id":"MDEyOk9yZ2FuaXphdGlvbjM2NTIzMA==","avatar_url":"https://avatars1.githubusercontent.com/u/365230?v=4","gravatar_id":"","url":"https://api.github.com/users/DataDog","html_url":"https://github.com/DataDog","followers_url":"https://api.github.com/users/DataDog/followers","following_url":"https://api.github.com/users/DataDog/following{/other_user}","gists_url":"https://api.github.com/users/DataDog/gists{/gist_id}","starred_url":"https://api.github.com/users/DataDog/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DataDog/subscriptions","organizations_url":"https://api.github.com/users/DataDog/orgs","repos_url":"https://api.github.com/users/DataDog/repos","events_url":"https://api.github.com/users/DataDog/events{/privacy}","received_events_url":"https://api.github.com/users/DataDog/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/DataDog/integrations-core","description":"Core integrations of the datadog agent","fork":false,"url":"https://api.github.com/repos/DataDog/integrations-core","forks_url":"https://api.github.com/repos/DataDog/integrations-core/forks","keys_url":"https://api.github.com/repos/DataDog/integrations-core/keys{/key_id}","collaborators_url":"https://api.github.com/repos/DataDog/integrations-core/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/DataDog/integrations-core/teams","hooks_url":"https://api.github.com/repos/DataDog/integrations-core/hooks","issue_events_url":"https://api.github.com/repos/DataDog/integrations-core/issues/events{/number}","events_url":"https://api.github.com/repos/DataDog/integrations-core/events","assignees_url":"https://api.github.com/repos/DataDog/integrations-core/assignees{/user}","branches_url":"https://api.github.com/repos/DataDog/integrations-core/branches{/branch}","tags_url":"https://api.github.com/repos/DataDog/integrations-core/tags","blobs_url":"https://api.github.com/repos/DataDog/integrations-core/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/DataDog/integrations-core/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/DataDog/integrations-core/git/refs{/sha}","trees_url":"https://api.github.com/repos/DataDog/integrations-core/git/trees{/sha}","statuses_url":"https://api.github.com/repos/DataDog/integrations-core/statuses/{sha}","languages_url":"https://api.github.com/repos/DataDog/integrations-core/languages","stargazers_url":"https://api.github.com/repos/DataDog/integrations-core/stargazers","contributors_url":"https://api.github.com/repos/DataDog/integrations-core/contributors","subscribers_url":"https://api.github.com/repos/DataDog/integrations-core/subscribers","subscription_url":"https://api.github.com/repos/DataDog/integrations-core/subscription","commits_url":"https://api.github.com/repos/DataDog/integrations-core/commits{/sha}","git_commits_url":"https://api.github.com/repos/DataDog/integrations-core/git/commits{/sha}","comments_url":"https://api.github.com/repos/DataDog/integrations-core/comments{/number}","issue_comment_url":"https://api.github.com/repos/DataDog/integrations-core/issues/comments{/number}","contents_url":"https://api.github.com/repos/DataDog/integrations-core/contents/{+path}","compare_url":"https://api.github.com/repos/DataDog/integrations-core/compare/{base}...{head}","merges_url":"https://api.github.com/repos/DataDog/integrations-core/merges","archive_url":"https://api.github.com/repos/DataDog/integrations-core/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/DataDog/integrations-core/downloads","issues_url":"https://api.github.com/repos/DataDog/integrations-core/issues{/number}","pulls_url":"https://api.github.com/repos/DataDog/integrations-core/pulls{/number}","milestones_url":"https://api.github.com/repos/DataDog/integrations-core/milestones{/number}","notifications_url":"https://api.github.com/repos/DataDog/integrations-core/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/DataDog/integrations-core/labels{/name}","releases_url":"https://api.github.com/repos/DataDog/integrations-core/releases{/id}","deployments_url":"https://api.github.com/repos/DataDog/integrations-core/deployments","created_at":"2015-12-01T16:41:45Z","updated_at":"2018-09-07T14:52:58Z","pushed_at":"2018-09-07T17:34:00Z","git_url":"git://github.com/DataDog/integrations-core.git","ssh_url":"git@github.com:DataDog/integrations-core.git","clone_url":"https://github.com/DataDog/integrations-core.git","svn_url":"https://github.com/DataDog/integrations-core","homepage":null,"size":28790,"stargazers_count":142,"watchers_count":142,"language":"Python","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":312,"mirror_url":null,"archived":false,"open_issues_count":61,"license":{"key":"bsd-3-clause","name":"BSD 3-Clause \"New\" or \"Revised\" License","spdx_id":"BSD-3-Clause","url":"https://api.github.com/licenses/bsd-3-clause","node_id":"MDc6TGljZW5zZTU="},"forks":312,"open_issues":61,"watchers":142,"default_branch":"master"}},"base":{"label":"DataDog:master","ref":"master","sha":"6f242387d362066f6ec22440809156332bdc7a1a","user":{"login":"DataDog","id":365230,"node_id":"MDEyOk9yZ2FuaXphdGlvbjM2NTIzMA==","avatar_url":"https://avatars1.githubusercontent.com/u/365230?v=4","gravatar_id":"","url":"https://api.github.com/users/DataDog","html_url":"https://github.com/DataDog","followers_url":"https://api.github.com/users/DataDog/followers","following_url":"https://api.github.com/users/DataDog/following{/other_user}","gists_url":"https://api.github.com/users/DataDog/gists{/gist_id}","starred_url":"https://api.github.com/users/DataDog/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DataDog/subscriptions","organizations_url":"https://api.github.com/users/DataDog/orgs","repos_url":"https://api.github.com/users/DataDog/repos","events_url":"https://api.github.com/users/DataDog/events{/privacy}","received_events_url":"https://api.github.com/users/DataDog/received_events","type":"Organization","site_admin":false},"repo":{"id":47203045,"node_id":"MDEwOlJlcG9zaXRvcnk0NzIwMzA0NQ==","name":"integrations-core","full_name":"DataDog/integrations-core","owner":{"login":"DataDog","id":365230,"node_id":"MDEyOk9yZ2FuaXphdGlvbjM2NTIzMA==","avatar_url":"https://avatars1.githubusercontent.com/u/365230?v=4","gravatar_id":"","url":"https://api.github.com/users/DataDog","html_url":"https://github.com/DataDog","followers_url":"https://api.github.com/users/DataDog/followers","following_url":"https://api.github.com/users/DataDog/following{/other_user}","gists_url":"https://api.github.com/users/DataDog/gists{/gist_id}","starred_url":"https://api.github.com/users/DataDog/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DataDog/subscriptions","organizations_url":"https://api.github.com/users/DataDog/orgs","repos_url":"https://api.github.com/users/DataDog/repos","events_url":"https://api.github.com/users/DataDog/events{/privacy}","received_events_url":"https://api.github.com/users/DataDog/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/DataDog/integrations-core","description":"Core integrations of the datadog agent","fork":false,"url":"https://api.github.com/repos/DataDog/integrations-core","forks_url":"https://api.github.com/repos/DataDog/integrations-core/forks","keys_url":"https://api.github.com/repos/DataDog/integrations-core/keys{/key_id}","collaborators_url":"https://api.github.com/repos/DataDog/integrations-core/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/DataDog/integrations-core/teams","hooks_url":"https://api.github.com/repos/DataDog/integrations-core/hooks","issue_events_url":"https://api.github.com/repos/DataDog/integrations-core/issues/events{/number}","events_url":"https://api.github.com/repos/DataDog/integrations-core/events","assignees_url":"https://api.github.com/repos/DataDog/integrations-core/assignees{/user}","branches_url":"https://api.github.com/repos/DataDog/integrations-core/branches{/branch}","tags_url":"https://api.github.com/repos/DataDog/integrations-core/tags","blobs_url":"https://api.github.com/repos/DataDog/integrations-core/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/DataDog/integrations-core/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/DataDog/integrations-core/git/refs{/sha}","trees_url":"https://api.github.com/repos/DataDog/integrations-core/git/trees{/sha}","statuses_url":"https://api.github.com/repos/DataDog/integrations-core/statuses/{sha}","languages_url":"https://api.github.com/repos/DataDog/integrations-core/languages","stargazers_url":"https://api.github.com/repos/DataDog/integrations-core/stargazers","contributors_url":"https://api.github.com/repos/DataDog/integrations-core/contributors","subscribers_url":"https://api.github.com/repos/DataDog/integrations-core/subscribers","subscription_url":"https://api.github.com/repos/DataDog/integrations-core/subscription","commits_url":"https://api.github.com/repos/DataDog/integrations-core/commits{/sha}","git_commits_url":"https://api.github.com/repos/DataDog/integrations-core/git/commits{/sha}","comments_url":"https://api.github.com/repos/DataDog/integrations-core/comments{/number}","issue_comment_url":"https://api.github.com/repos/DataDog/integrations-core/issues/comments{/number}","contents_url":"https://api.github.com/repos/DataDog/integrations-core/contents/{+path}","compare_url":"https://api.github.com/repos/DataDog/integrations-core/compare/{base}...{head}","merges_url":"https://api.github.com/repos/DataDog/integrations-core/merges","archive_url":"https://api.github.com/repos/DataDog/integrations-core/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/DataDog/integrations-core/downloads","issues_url":"https://api.github.com/repos/DataDog/integrations-core/issues{/number}","pulls_url":"https://api.github.com/repos/DataDog/integrations-core/pulls{/number}","milestones_url":"https://api.github.com/repos/DataDog/integrations-core/milestones{/number}","notifications_url":"https://api.github.com/repos/DataDog/integrations-core/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/DataDog/integrations-core/labels{/name}","releases_url":"https://api.github.com/repos/DataDog/integrations-core/releases{/id}","deployments_url":"https://api.github.com/repos/DataDog/integrations-core/deployments","created_at":"2015-12-01T16:41:45Z","updated_at":"2018-09-07T14:52:58Z","pushed_at":"2018-09-07T17:34:00Z","git_url":"git://github.com/DataDog/integrations-core.git","ssh_url":"git@github.com:DataDog/integrations-core.git","clone_url":"https://github.com/DataDog/integrations-core.git","svn_url":"https://github.com/DataDog/integrations-core","homepage":null,"size":28790,"stargazers_count":142,"watchers_count":142,"language":"Python","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":312,"mirror_url":null,"archived":false,"open_issues_count":61,"license":{"key":"bsd-3-clause","name":"BSD 3-Clause \"New\" or \"Revised\" License","spdx_id":"BSD-3-Clause","url":"https://api.github.com/licenses/bsd-3-clause","node_id":"MDc6TGljZW5zZTU="},"forks":312,"open_issues":61,"watchers":142,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/DataDog/integrations-core/pulls/2161"},"html":{"href":"https://github.com/DataDog/integrations-core/pull/2161"},"issue":{"href":"https://api.github.com/repos/DataDog/integrations-core/issues/2161"},"comments":{"href":"https://api.github.com/repos/DataDog/integrations-core/issues/2161/comments"},"review_comments":{"href":"https://api.github.com/repos/DataDog/integrations-core/pulls/2161/comments"},"review_comment":{"href":"https://api.github.com/repos/DataDog/integrations-core/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/DataDog/integrations-core/pulls/2161/commits"},"statuses":{"href":"https://api.github.com/repos/DataDog/integrations-core/statuses/96cb93de3ae49f06e8cd3f78223e3a8d378d8825"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"nmuesch"}}