{"action":"created","comment":{"url":"https://api.github.com/repos/DataDog/dd-agent/pulls/comments/57249788","id":57249788,"diff_hunk":"@@ -500,14 +515,38 @@ def self_destruct(signum, frame):\n         finally:\n             os.kill(os.getpid(), signal.SIGKILL)\n \n+    def is_frenetic(self):","path":"util.py","position":60,"original_position":60,"commit_id":"09db8cb12a9dd0f57b8a1865bba5b5ff33fd5d7d","original_commit_id":"09db8cb12a9dd0f57b8a1865bba5b5ff33fd5d7d","user":{"login":"olivielpeau","id":4912500,"avatar_url":"https://avatars.githubusercontent.com/u/4912500?v=3","gravatar_id":"","url":"https://api.github.com/users/olivielpeau","html_url":"https://github.com/olivielpeau","followers_url":"https://api.github.com/users/olivielpeau/followers","following_url":"https://api.github.com/users/olivielpeau/following{/other_user}","gists_url":"https://api.github.com/users/olivielpeau/gists{/gist_id}","starred_url":"https://api.github.com/users/olivielpeau/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/olivielpeau/subscriptions","organizations_url":"https://api.github.com/users/olivielpeau/orgs","repos_url":"https://api.github.com/users/olivielpeau/repos","events_url":"https://api.github.com/users/olivielpeau/events{/privacy}","received_events_url":"https://api.github.com/users/olivielpeau/received_events","type":"User","site_admin":false},"body":"nitpick: this method is \"private\" so should rather be named `_is_frenetic`","created_at":"2016-03-23T22:40:59Z","updated_at":"2016-03-23T22:40:59Z","html_url":"https://github.com/DataDog/dd-agent/pull/2322#discussion_r57249788","pull_request_url":"https://api.github.com/repos/DataDog/dd-agent/pulls/2322","_links":{"self":{"href":"https://api.github.com/repos/DataDog/dd-agent/pulls/comments/57249788"},"html":{"href":"https://github.com/DataDog/dd-agent/pull/2322#discussion_r57249788"},"pull_request":{"href":"https://api.github.com/repos/DataDog/dd-agent/pulls/2322"}}},"pull_request":{"url":"https://api.github.com/repos/DataDog/dd-agent/pulls/2322","id":61753699,"html_url":"https://github.com/DataDog/dd-agent/pull/2322","diff_url":"https://github.com/DataDog/dd-agent/pull/2322.diff","patch_url":"https://github.com/DataDog/dd-agent/pull/2322.patch","issue_url":"https://api.github.com/repos/DataDog/dd-agent/issues/2322","number":2322,"state":"open","locked":false,"title":"[core] detect forwarder pathological activity","user":{"login":"yannmh","id":5708089,"avatar_url":"https://avatars.githubusercontent.com/u/5708089?v=3","gravatar_id":"","url":"https://api.github.com/users/yannmh","html_url":"https://github.com/yannmh","followers_url":"https://api.github.com/users/yannmh/followers","following_url":"https://api.github.com/users/yannmh/following{/other_user}","gists_url":"https://api.github.com/users/yannmh/gists{/gist_id}","starred_url":"https://api.github.com/users/yannmh/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yannmh/subscriptions","organizations_url":"https://api.github.com/users/yannmh/orgs","repos_url":"https://api.github.com/users/yannmh/repos","events_url":"https://api.github.com/users/yannmh/events{/privacy}","received_events_url":"https://api.github.com/users/yannmh/received_events","type":"User","site_admin":false},"body":"**[util] watchdog high activity detection :fire:**\r\n\r\nNew option to the Watchdog to detect suspiciously high activity:\r\nrestarts the process when too many resets were made for a given\r\ntimeframe.\r\n\r\n**[core] detect forwarder pathological activity**\r\nOn some specific system date and scheduled transaction flush time, the\r\nDatadog Agent forwarder's schedule breaks and enters in a frenetic loop.\r\nIt gets unreachable, flushes (no transaction) at a very high rate\r\n(several ~100 flushes/second). At this point, only a restart fixes the\r\nsituation.\r\n\r\nUnder the hood, the issue is related to `tornado==3.2.2`. More\r\ninformation is available here:\r\nhttps://github.com/tornadoweb/tornado/issues/947\r\n\r\nBefore considering any upgrade, a simple fix is to use the Watchdog to\r\ndetect this pathological high activity.","created_at":"2016-03-04T18:47:17Z","updated_at":"2016-03-23T22:40:59Z","closed_at":null,"merged_at":null,"merge_commit_sha":"8962f53055c0228cec66f90889960bd1ca2ea05b","assignee":{"login":"olivielpeau","id":4912500,"avatar_url":"https://avatars.githubusercontent.com/u/4912500?v=3","gravatar_id":"","url":"https://api.github.com/users/olivielpeau","html_url":"https://github.com/olivielpeau","followers_url":"https://api.github.com/users/olivielpeau/followers","following_url":"https://api.github.com/users/olivielpeau/following{/other_user}","gists_url":"https://api.github.com/users/olivielpeau/gists{/gist_id}","starred_url":"https://api.github.com/users/olivielpeau/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/olivielpeau/subscriptions","organizations_url":"https://api.github.com/users/olivielpeau/orgs","repos_url":"https://api.github.com/users/olivielpeau/repos","events_url":"https://api.github.com/users/olivielpeau/events{/privacy}","received_events_url":"https://api.github.com/users/olivielpeau/received_events","type":"User","site_admin":false},"milestone":{"url":"https://api.github.com/repos/DataDog/dd-agent/milestones/65","html_url":"https://github.com/DataDog/dd-agent/milestones/5.7.3","labels_url":"https://api.github.com/repos/DataDog/dd-agent/milestones/65/labels","id":1658300,"number":65,"title":"5.7.3","description":"","creator":{"login":"olivielpeau","id":4912500,"avatar_url":"https://avatars.githubusercontent.com/u/4912500?v=3","gravatar_id":"","url":"https://api.github.com/users/olivielpeau","html_url":"https://github.com/olivielpeau","followers_url":"https://api.github.com/users/olivielpeau/followers","following_url":"https://api.github.com/users/olivielpeau/following{/other_user}","gists_url":"https://api.github.com/users/olivielpeau/gists{/gist_id}","starred_url":"https://api.github.com/users/olivielpeau/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/olivielpeau/subscriptions","organizations_url":"https://api.github.com/users/olivielpeau/orgs","repos_url":"https://api.github.com/users/olivielpeau/repos","events_url":"https://api.github.com/users/olivielpeau/events{/privacy}","received_events_url":"https://api.github.com/users/olivielpeau/received_events","type":"User","site_admin":false},"open_issues":7,"closed_issues":2,"state":"open","created_at":"2016-03-21T19:52:08Z","updated_at":"2016-03-23T21:36:39Z","due_on":null,"closed_at":null},"commits_url":"https://api.github.com/repos/DataDog/dd-agent/pulls/2322/commits","review_comments_url":"https://api.github.com/repos/DataDog/dd-agent/pulls/2322/comments","review_comment_url":"https://api.github.com/repos/DataDog/dd-agent/pulls/comments{/number}","comments_url":"https://api.github.com/repos/DataDog/dd-agent/issues/2322/comments","statuses_url":"https://api.github.com/repos/DataDog/dd-agent/statuses/09db8cb12a9dd0f57b8a1865bba5b5ff33fd5d7d","head":{"label":"DataDog:yann/watchdog-frenesy-detection","ref":"yann/watchdog-frenesy-detection","sha":"09db8cb12a9dd0f57b8a1865bba5b5ff33fd5d7d","user":{"login":"DataDog","id":365230,"avatar_url":"https://avatars.githubusercontent.com/u/365230?v=3","gravatar_id":"","url":"https://api.github.com/users/DataDog","html_url":"https://github.com/DataDog","followers_url":"https://api.github.com/users/DataDog/followers","following_url":"https://api.github.com/users/DataDog/following{/other_user}","gists_url":"https://api.github.com/users/DataDog/gists{/gist_id}","starred_url":"https://api.github.com/users/DataDog/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DataDog/subscriptions","organizations_url":"https://api.github.com/users/DataDog/orgs","repos_url":"https://api.github.com/users/DataDog/repos","events_url":"https://api.github.com/users/DataDog/events{/privacy}","received_events_url":"https://api.github.com/users/DataDog/received_events","type":"Organization","site_admin":false},"repo":{"id":1210071,"name":"dd-agent","full_name":"DataDog/dd-agent","owner":{"login":"DataDog","id":365230,"avatar_url":"https://avatars.githubusercontent.com/u/365230?v=3","gravatar_id":"","url":"https://api.github.com/users/DataDog","html_url":"https://github.com/DataDog","followers_url":"https://api.github.com/users/DataDog/followers","following_url":"https://api.github.com/users/DataDog/following{/other_user}","gists_url":"https://api.github.com/users/DataDog/gists{/gist_id}","starred_url":"https://api.github.com/users/DataDog/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DataDog/subscriptions","organizations_url":"https://api.github.com/users/DataDog/orgs","repos_url":"https://api.github.com/users/DataDog/repos","events_url":"https://api.github.com/users/DataDog/events{/privacy}","received_events_url":"https://api.github.com/users/DataDog/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/DataDog/dd-agent","description":"Datadog Agent","fork":false,"url":"https://api.github.com/repos/DataDog/dd-agent","forks_url":"https://api.github.com/repos/DataDog/dd-agent/forks","keys_url":"https://api.github.com/repos/DataDog/dd-agent/keys{/key_id}","collaborators_url":"https://api.github.com/repos/DataDog/dd-agent/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/DataDog/dd-agent/teams","hooks_url":"https://api.github.com/repos/DataDog/dd-agent/hooks","issue_events_url":"https://api.github.com/repos/DataDog/dd-agent/issues/events{/number}","events_url":"https://api.github.com/repos/DataDog/dd-agent/events","assignees_url":"https://api.github.com/repos/DataDog/dd-agent/assignees{/user}","branches_url":"https://api.github.com/repos/DataDog/dd-agent/branches{/branch}","tags_url":"https://api.github.com/repos/DataDog/dd-agent/tags","blobs_url":"https://api.github.com/repos/DataDog/dd-agent/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/DataDog/dd-agent/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/DataDog/dd-agent/git/refs{/sha}","trees_url":"https://api.github.com/repos/DataDog/dd-agent/git/trees{/sha}","statuses_url":"https://api.github.com/repos/DataDog/dd-agent/statuses/{sha}","languages_url":"https://api.github.com/repos/DataDog/dd-agent/languages","stargazers_url":"https://api.github.com/repos/DataDog/dd-agent/stargazers","contributors_url":"https://api.github.com/repos/DataDog/dd-agent/contributors","subscribers_url":"https://api.github.com/repos/DataDog/dd-agent/subscribers","subscription_url":"https://api.github.com/repos/DataDog/dd-agent/subscription","commits_url":"https://api.github.com/repos/DataDog/dd-agent/commits{/sha}","git_commits_url":"https://api.github.com/repos/DataDog/dd-agent/git/commits{/sha}","comments_url":"https://api.github.com/repos/DataDog/dd-agent/comments{/number}","issue_comment_url":"https://api.github.com/repos/DataDog/dd-agent/issues/comments{/number}","contents_url":"https://api.github.com/repos/DataDog/dd-agent/contents/{+path}","compare_url":"https://api.github.com/repos/DataDog/dd-agent/compare/{base}...{head}","merges_url":"https://api.github.com/repos/DataDog/dd-agent/merges","archive_url":"https://api.github.com/repos/DataDog/dd-agent/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/DataDog/dd-agent/downloads","issues_url":"https://api.github.com/repos/DataDog/dd-agent/issues{/number}","pulls_url":"https://api.github.com/repos/DataDog/dd-agent/pulls{/number}","milestones_url":"https://api.github.com/repos/DataDog/dd-agent/milestones{/number}","notifications_url":"https://api.github.com/repos/DataDog/dd-agent/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/DataDog/dd-agent/labels{/name}","releases_url":"https://api.github.com/repos/DataDog/dd-agent/releases{/id}","deployments_url":"https://api.github.com/repos/DataDog/dd-agent/deployments","created_at":"2010-12-31T03:02:47Z","updated_at":"2016-03-22T01:01:47Z","pushed_at":"2016-03-23T22:24:24Z","git_url":"git://github.com/DataDog/dd-agent.git","ssh_url":"git@github.com:DataDog/dd-agent.git","clone_url":"https://github.com/DataDog/dd-agent.git","svn_url":"https://github.com/DataDog/dd-agent","homepage":"http://docs.datadoghq.com/","size":40301,"stargazers_count":460,"watchers_count":460,"language":"Python","has_issues":true,"has_downloads":false,"has_wiki":true,"has_pages":true,"forks_count":422,"mirror_url":null,"open_issues_count":163,"forks":422,"open_issues":163,"watchers":460,"default_branch":"master"}},"base":{"label":"DataDog:master","ref":"master","sha":"4852130c3cd6228c9d0b22f46b009ee2fc7becaa","user":{"login":"DataDog","id":365230,"avatar_url":"https://avatars.githubusercontent.com/u/365230?v=3","gravatar_id":"","url":"https://api.github.com/users/DataDog","html_url":"https://github.com/DataDog","followers_url":"https://api.github.com/users/DataDog/followers","following_url":"https://api.github.com/users/DataDog/following{/other_user}","gists_url":"https://api.github.com/users/DataDog/gists{/gist_id}","starred_url":"https://api.github.com/users/DataDog/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DataDog/subscriptions","organizations_url":"https://api.github.com/users/DataDog/orgs","repos_url":"https://api.github.com/users/DataDog/repos","events_url":"https://api.github.com/users/DataDog/events{/privacy}","received_events_url":"https://api.github.com/users/DataDog/received_events","type":"Organization","site_admin":false},"repo":{"id":1210071,"name":"dd-agent","full_name":"DataDog/dd-agent","owner":{"login":"DataDog","id":365230,"avatar_url":"https://avatars.githubusercontent.com/u/365230?v=3","gravatar_id":"","url":"https://api.github.com/users/DataDog","html_url":"https://github.com/DataDog","followers_url":"https://api.github.com/users/DataDog/followers","following_url":"https://api.github.com/users/DataDog/following{/other_user}","gists_url":"https://api.github.com/users/DataDog/gists{/gist_id}","starred_url":"https://api.github.com/users/DataDog/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DataDog/subscriptions","organizations_url":"https://api.github.com/users/DataDog/orgs","repos_url":"https://api.github.com/users/DataDog/repos","events_url":"https://api.github.com/users/DataDog/events{/privacy}","received_events_url":"https://api.github.com/users/DataDog/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/DataDog/dd-agent","description":"Datadog Agent","fork":false,"url":"https://api.github.com/repos/DataDog/dd-agent","forks_url":"https://api.github.com/repos/DataDog/dd-agent/forks","keys_url":"https://api.github.com/repos/DataDog/dd-agent/keys{/key_id}","collaborators_url":"https://api.github.com/repos/DataDog/dd-agent/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/DataDog/dd-agent/teams","hooks_url":"https://api.github.com/repos/DataDog/dd-agent/hooks","issue_events_url":"https://api.github.com/repos/DataDog/dd-agent/issues/events{/number}","events_url":"https://api.github.com/repos/DataDog/dd-agent/events","assignees_url":"https://api.github.com/repos/DataDog/dd-agent/assignees{/user}","branches_url":"https://api.github.com/repos/DataDog/dd-agent/branches{/branch}","tags_url":"https://api.github.com/repos/DataDog/dd-agent/tags","blobs_url":"https://api.github.com/repos/DataDog/dd-agent/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/DataDog/dd-agent/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/DataDog/dd-agent/git/refs{/sha}","trees_url":"https://api.github.com/repos/DataDog/dd-agent/git/trees{/sha}","statuses_url":"https://api.github.com/repos/DataDog/dd-agent/statuses/{sha}","languages_url":"https://api.github.com/repos/DataDog/dd-agent/languages","stargazers_url":"https://api.github.com/repos/DataDog/dd-agent/stargazers","contributors_url":"https://api.github.com/repos/DataDog/dd-agent/contributors","subscribers_url":"https://api.github.com/repos/DataDog/dd-agent/subscribers","subscription_url":"https://api.github.com/repos/DataDog/dd-agent/subscription","commits_url":"https://api.github.com/repos/DataDog/dd-agent/commits{/sha}","git_commits_url":"https://api.github.com/repos/DataDog/dd-agent/git/commits{/sha}","comments_url":"https://api.github.com/repos/DataDog/dd-agent/comments{/number}","issue_comment_url":"https://api.github.com/repos/DataDog/dd-agent/issues/comments{/number}","contents_url":"https://api.github.com/repos/DataDog/dd-agent/contents/{+path}","compare_url":"https://api.github.com/repos/DataDog/dd-agent/compare/{base}...{head}","merges_url":"https://api.github.com/repos/DataDog/dd-agent/merges","archive_url":"https://api.github.com/repos/DataDog/dd-agent/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/DataDog/dd-agent/downloads","issues_url":"https://api.github.com/repos/DataDog/dd-agent/issues{/number}","pulls_url":"https://api.github.com/repos/DataDog/dd-agent/pulls{/number}","milestones_url":"https://api.github.com/repos/DataDog/dd-agent/milestones{/number}","notifications_url":"https://api.github.com/repos/DataDog/dd-agent/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/DataDog/dd-agent/labels{/name}","releases_url":"https://api.github.com/repos/DataDog/dd-agent/releases{/id}","deployments_url":"https://api.github.com/repos/DataDog/dd-agent/deployments","created_at":"2010-12-31T03:02:47Z","updated_at":"2016-03-22T01:01:47Z","pushed_at":"2016-03-23T22:24:24Z","git_url":"git://github.com/DataDog/dd-agent.git","ssh_url":"git@github.com:DataDog/dd-agent.git","clone_url":"https://github.com/DataDog/dd-agent.git","svn_url":"https://github.com/DataDog/dd-agent","homepage":"http://docs.datadoghq.com/","size":40301,"stargazers_count":460,"watchers_count":460,"language":"Python","has_issues":true,"has_downloads":false,"has_wiki":true,"has_pages":true,"forks_count":422,"mirror_url":null,"open_issues_count":163,"forks":422,"open_issues":163,"watchers":460,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/DataDog/dd-agent/pulls/2322"},"html":{"href":"https://github.com/DataDog/dd-agent/pull/2322"},"issue":{"href":"https://api.github.com/repos/DataDog/dd-agent/issues/2322"},"comments":{"href":"https://api.github.com/repos/DataDog/dd-agent/issues/2322/comments"},"review_comments":{"href":"https://api.github.com/repos/DataDog/dd-agent/pulls/2322/comments"},"review_comment":{"href":"https://api.github.com/repos/DataDog/dd-agent/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/DataDog/dd-agent/pulls/2322/commits"},"statuses":{"href":"https://api.github.com/repos/DataDog/dd-agent/statuses/09db8cb12a9dd0f57b8a1865bba5b5ff33fd5d7d"}}}}