{"action":"created","comment":{"url":"https://api.github.com/repos/pingcap/oasis/pulls/comments/180949906","pull_request_review_id":111461101,"id":180949906,"diff_hunk":"@@ -2,15 +2,153 @@\n \n from __future__ import absolute_import\n \n-from oasis.models.model import Model\n+import datetime\n+import time\n+from threading import Thread, Timer\n+from pytimeparse.timeparse import timeparse\n+from oasis.models.model import Model, Config\n from oasis.datasource import PrometheusAPI, PrometheusQuery, Metrics\n+from oasis.libs.log import logger\n+from oasis.models.util import sub_id, EPS\n+from oasis.libs.features import Features\n+from oasis.libs.alert import send_to_slack\n+from oasis.libs.rules import parse_rule\n \n \n class Rules(Model):\n-    def __init__(self, job, callback):\n+    def __init__(self, job):\n+        super(Rules, self).__init__()\n         self.job = job\n-        self.callback = callback\n         self.api = PrometheusAPI(job.data_source)\n+        self.timer = Timer(timeparse(self.job.timeout), self.timeout_action)\n+        self.config_file = \"%s/rules.yml\" % self.model_path\n+        self.cfg = RulesConfig(self.config_file, job.config)\n \n+    def run(self):\n+        self.timer.start()\n+        for key in self.job.metrics:\n+            if key in Metrics:\n+                val = Metrics[key]\n+                if key not in self.cfg.metrics:\n+                    continue\n \n+                t = Thread(target=self.run_action,\n+                           args=(key, val, self.cfg.metrics[key],\n+                                 self.cfg.rules[key]))\n+                t.start()\n+                self.threads[key] = t\n \n+    def run_action(self, metric, val, config, rules):\n+        logger.info(\"[job-id:{id}][metric:{metric}] start ot run\"\n+                    .format(id=sub_id(self.job.id), metric=metric))\n+        self.compute(metric, val, config, rules)\n+\n+    def compute(self, metric, query_expr, config, rules):\n+        # extraction features and match with rules\n+        while not self._exit:\n+            data_set = self.query_data(query_expr)\n+            if len(data_set) > 0:\n+                features_value = self.extraction_features(data_set, config)\n+                logger.info(\"[job-id:{id}][metric:{metric}] extraction features {value}, \"\n+                            \"start to match with rule\"\n+                            .format(id=sub_id(self.job.id),\n+                                    metric=metric, value=features_value))\n+                self.match_rules(metric, features_value, rules)\n+\n+            self.event.wait(timeparse(self.cfg.model[\"predict_interval\"]))\n+\n+        logger.info(\"[job-id:{id}][metric:{metric}] stop\"\n+                    .format(id=sub_id(self.job.id), metric=metric))\n+\n+    def query_data(self, query_expr):\n+        now = datetime.datetime.now()\n+        query = PrometheusQuery(query_expr,\n+                                time.mktime((now - datetime.timedelta(\n+                                    seconds=timeparse(self.cfg.model[\"data_range\"])))\n+                                            .timetuple()),\n+                                time.mktime(now.timetuple()), \"15s\")\n+        return self.api.query(query)\n+\n+    def extraction_features(self, data_set, config):\n+        values = []\n+        for data in data_set.values():\n+            values.append(float(data))\n+\n+        features_value = {}\n+        for key in config[\"features\"]:\n+            if key in Features:\n+                features_value[key] = Features[key](values)\n+\n+        return features_value\n+\n+    def match_rules(self, metric, features_value, rules):\n+        match_flag = True\n+        for rule in rules:\n+            if rule[\"feature\"] not in features_value:\n+                continue\n+\n+            if check_is_triggered(features_value[rule[\"feature\"]],\n+                                  rule[\"operator\"], rule[\"value\"]):\n+                logger.error(\"[job-id:{id}][metric:{metric}] not match rule: {rule}\"\n+                             .format(id=sub_id(self.job.id), metric=metric, rule=rule))\n+                send_to_slack(\"[job] {job}, metric: {metric} not match rule: {rule}\"\n+                              .format(job=dict(self.job), metric=metric,\n+                                      rule=rule), self.job.slack_channel)\n+                match_flag = False\n+                break\n+\n+        if match_flag:\n+            logger.info(\"[job-id:{id}][metric:{metric}] predict OK\"\n+                        .format(id=sub_id(self.job.id), metric=metric))\n+\n+    def close(self):\n+        logger.info(\"[job-id:{id}] closing the job\"\n+                    .format(id=sub_id(self.job.id)))\n+        super(Rules, self).close()\n+        self.timer.cancel()\n+\n+    def timeout_action(self):\n+        logger.info(\"[job-id:{id}] finish the job\"\n+                    .format(id=sub_id(self.job.id)))\n+        super(Rules, self).close()\n+\n+\n+class RulesConfig(Config):\n+    def __init__(self, config_file, config_json=None):\n+        super(RulesConfig, self).__init__(config_file, config_json)\n+        self._init()\n+        self.rules = dict()\n+        self._parse_rules()\n+\n+    def _set_default_config(self):\n+        self.model.setdefault(\"data_range\", \"10m\")\n+        self.model.setdefault(\"predict_interval\", \"5m\")\n+\n+        for metric in Metrics.keys():\n+            self.metrics[metric] = {\n+                \"features\": [\"mean\", \"std\"],\n+                \"rules\": [\"features[std] > 1000\"]\n+            }\n+\n+    def _parse_rules(self):\n+        for metric in self.metrics:\n+            rules = self.metrics[metric].get(\"rules\")\n+            self.rules[metric] = parse_rule(rules)\n+\n+\n+def check_is_triggered(left_value, operator, right_value):\n+    operators = {\n+        \"==\": lambda: abs(left_value-right_value) < EPS,\n+        \"!=\": lambda: abs(left_value-right_value) > EPS,\n+        \"<=\": lambda: left_value <= right_value,\n+        \">=\": lambda: left_value >= right_value,\n+        \"=\": lambda: abs(left_value-right_value) < EPS,\n+        \"<\": lambda: left_value < right_value,\n+        \">\": lambda: left_value > right_value,\n+    }\n+\n+    if operator not in operators:\n+        logger.error(\"operator {} is invalid\".format(operator))\n+        return False\n+\n+    return operators[operator]","path":"oasis/models/rules.py","position":156,"original_position":156,"commit_id":"13b70d12e584673e8a4acbed7d8b85095e22f805","original_commit_id":"13b70d12e584673e8a4acbed7d8b85095e22f805","user":{"login":"foreverbell","id":3240173,"avatar_url":"https://avatars3.githubusercontent.com/u/3240173?v=4","gravatar_id":"","url":"https://api.github.com/users/foreverbell","html_url":"https://github.com/foreverbell","followers_url":"https://api.github.com/users/foreverbell/followers","following_url":"https://api.github.com/users/foreverbell/following{/other_user}","gists_url":"https://api.github.com/users/foreverbell/gists{/gist_id}","starred_url":"https://api.github.com/users/foreverbell/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/foreverbell/subscriptions","organizations_url":"https://api.github.com/users/foreverbell/orgs","repos_url":"https://api.github.com/users/foreverbell/repos","events_url":"https://api.github.com/users/foreverbell/events{/privacy}","received_events_url":"https://api.github.com/users/foreverbell/received_events","type":"User","site_admin":false},"body":"`operators[operator]()`, otherwise we return a lambda here.","created_at":"2018-04-12T02:29:00Z","updated_at":"2018-04-12T02:31:30Z","html_url":"https://github.com/pingcap/oasis/pull/4#discussion_r180949906","pull_request_url":"https://api.github.com/repos/pingcap/oasis/pulls/4","author_association":"NONE","_links":{"self":{"href":"https://api.github.com/repos/pingcap/oasis/pulls/comments/180949906"},"html":{"href":"https://github.com/pingcap/oasis/pull/4#discussion_r180949906"},"pull_request":{"href":"https://api.github.com/repos/pingcap/oasis/pulls/4"}}},"pull_request":{"url":"https://api.github.com/repos/pingcap/oasis/pulls/4","id":180816193,"html_url":"https://github.com/pingcap/oasis/pull/4","diff_url":"https://github.com/pingcap/oasis/pull/4.diff","patch_url":"https://github.com/pingcap/oasis/pull/4.patch","issue_url":"https://api.github.com/repos/pingcap/oasis/issues/4","number":4,"state":"open","locked":false,"title":"add rules model","user":{"login":"cwen0","id":22956341,"avatar_url":"https://avatars3.githubusercontent.com/u/22956341?v=4","gravatar_id":"","url":"https://api.github.com/users/cwen0","html_url":"https://github.com/cwen0","followers_url":"https://api.github.com/users/cwen0/followers","following_url":"https://api.github.com/users/cwen0/following{/other_user}","gists_url":"https://api.github.com/users/cwen0/gists{/gist_id}","starred_url":"https://api.github.com/users/cwen0/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cwen0/subscriptions","organizations_url":"https://api.github.com/users/cwen0/orgs","repos_url":"https://api.github.com/users/cwen0/repos","events_url":"https://api.github.com/users/cwen0/events{/privacy}","received_events_url":"https://api.github.com/users/cwen0/received_events","type":"User","site_admin":false},"body":"@zhouqiang-cl @foreverbell PTAL","created_at":"2018-04-11T07:46:35Z","updated_at":"2018-04-12T02:31:30Z","closed_at":null,"merged_at":null,"merge_commit_sha":"df5ef11052bf339a48a7fa27fb80b83ec93c235a","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/pingcap/oasis/pulls/4/commits","review_comments_url":"https://api.github.com/repos/pingcap/oasis/pulls/4/comments","review_comment_url":"https://api.github.com/repos/pingcap/oasis/pulls/comments{/number}","comments_url":"https://api.github.com/repos/pingcap/oasis/issues/4/comments","statuses_url":"https://api.github.com/repos/pingcap/oasis/statuses/13b70d12e584673e8a4acbed7d8b85095e22f805","head":{"label":"pingcap:cwen/add_rules_model","ref":"cwen/add_rules_model","sha":"13b70d12e584673e8a4acbed7d8b85095e22f805","user":{"login":"pingcap","id":11855343,"avatar_url":"https://avatars3.githubusercontent.com/u/11855343?v=4","gravatar_id":"","url":"https://api.github.com/users/pingcap","html_url":"https://github.com/pingcap","followers_url":"https://api.github.com/users/pingcap/followers","following_url":"https://api.github.com/users/pingcap/following{/other_user}","gists_url":"https://api.github.com/users/pingcap/gists{/gist_id}","starred_url":"https://api.github.com/users/pingcap/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pingcap/subscriptions","organizations_url":"https://api.github.com/users/pingcap/orgs","repos_url":"https://api.github.com/users/pingcap/repos","events_url":"https://api.github.com/users/pingcap/events{/privacy}","received_events_url":"https://api.github.com/users/pingcap/received_events","type":"Organization","site_admin":false},"repo":{"id":127717062,"name":"oasis","full_name":"pingcap/oasis","owner":{"login":"pingcap","id":11855343,"avatar_url":"https://avatars3.githubusercontent.com/u/11855343?v=4","gravatar_id":"","url":"https://api.github.com/users/pingcap","html_url":"https://github.com/pingcap","followers_url":"https://api.github.com/users/pingcap/followers","following_url":"https://api.github.com/users/pingcap/following{/other_user}","gists_url":"https://api.github.com/users/pingcap/gists{/gist_id}","starred_url":"https://api.github.com/users/pingcap/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pingcap/subscriptions","organizations_url":"https://api.github.com/users/pingcap/orgs","repos_url":"https://api.github.com/users/pingcap/repos","events_url":"https://api.github.com/users/pingcap/events{/privacy}","received_events_url":"https://api.github.com/users/pingcap/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/pingcap/oasis","description":null,"fork":false,"url":"https://api.github.com/repos/pingcap/oasis","forks_url":"https://api.github.com/repos/pingcap/oasis/forks","keys_url":"https://api.github.com/repos/pingcap/oasis/keys{/key_id}","collaborators_url":"https://api.github.com/repos/pingcap/oasis/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/pingcap/oasis/teams","hooks_url":"https://api.github.com/repos/pingcap/oasis/hooks","issue_events_url":"https://api.github.com/repos/pingcap/oasis/issues/events{/number}","events_url":"https://api.github.com/repos/pingcap/oasis/events","assignees_url":"https://api.github.com/repos/pingcap/oasis/assignees{/user}","branches_url":"https://api.github.com/repos/pingcap/oasis/branches{/branch}","tags_url":"https://api.github.com/repos/pingcap/oasis/tags","blobs_url":"https://api.github.com/repos/pingcap/oasis/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/pingcap/oasis/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/pingcap/oasis/git/refs{/sha}","trees_url":"https://api.github.com/repos/pingcap/oasis/git/trees{/sha}","statuses_url":"https://api.github.com/repos/pingcap/oasis/statuses/{sha}","languages_url":"https://api.github.com/repos/pingcap/oasis/languages","stargazers_url":"https://api.github.com/repos/pingcap/oasis/stargazers","contributors_url":"https://api.github.com/repos/pingcap/oasis/contributors","subscribers_url":"https://api.github.com/repos/pingcap/oasis/subscribers","subscription_url":"https://api.github.com/repos/pingcap/oasis/subscription","commits_url":"https://api.github.com/repos/pingcap/oasis/commits{/sha}","git_commits_url":"https://api.github.com/repos/pingcap/oasis/git/commits{/sha}","comments_url":"https://api.github.com/repos/pingcap/oasis/comments{/number}","issue_comment_url":"https://api.github.com/repos/pingcap/oasis/issues/comments{/number}","contents_url":"https://api.github.com/repos/pingcap/oasis/contents/{+path}","compare_url":"https://api.github.com/repos/pingcap/oasis/compare/{base}...{head}","merges_url":"https://api.github.com/repos/pingcap/oasis/merges","archive_url":"https://api.github.com/repos/pingcap/oasis/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/pingcap/oasis/downloads","issues_url":"https://api.github.com/repos/pingcap/oasis/issues{/number}","pulls_url":"https://api.github.com/repos/pingcap/oasis/pulls{/number}","milestones_url":"https://api.github.com/repos/pingcap/oasis/milestones{/number}","notifications_url":"https://api.github.com/repos/pingcap/oasis/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/pingcap/oasis/labels{/name}","releases_url":"https://api.github.com/repos/pingcap/oasis/releases{/id}","deployments_url":"https://api.github.com/repos/pingcap/oasis/deployments","created_at":"2018-04-02T06:54:14Z","updated_at":"2018-04-11T05:44:50Z","pushed_at":"2018-04-12T02:15:16Z","git_url":"git://github.com/pingcap/oasis.git","ssh_url":"git@github.com:pingcap/oasis.git","clone_url":"https://github.com/pingcap/oasis.git","svn_url":"https://github.com/pingcap/oasis","homepage":null,"size":48,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":2,"license":null,"forks":0,"open_issues":2,"watchers":0,"default_branch":"master"}},"base":{"label":"pingcap:master","ref":"master","sha":"fa2e20c376513f73766a79d96f9f837e28da2e2d","user":{"login":"pingcap","id":11855343,"avatar_url":"https://avatars3.githubusercontent.com/u/11855343?v=4","gravatar_id":"","url":"https://api.github.com/users/pingcap","html_url":"https://github.com/pingcap","followers_url":"https://api.github.com/users/pingcap/followers","following_url":"https://api.github.com/users/pingcap/following{/other_user}","gists_url":"https://api.github.com/users/pingcap/gists{/gist_id}","starred_url":"https://api.github.com/users/pingcap/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pingcap/subscriptions","organizations_url":"https://api.github.com/users/pingcap/orgs","repos_url":"https://api.github.com/users/pingcap/repos","events_url":"https://api.github.com/users/pingcap/events{/privacy}","received_events_url":"https://api.github.com/users/pingcap/received_events","type":"Organization","site_admin":false},"repo":{"id":127717062,"name":"oasis","full_name":"pingcap/oasis","owner":{"login":"pingcap","id":11855343,"avatar_url":"https://avatars3.githubusercontent.com/u/11855343?v=4","gravatar_id":"","url":"https://api.github.com/users/pingcap","html_url":"https://github.com/pingcap","followers_url":"https://api.github.com/users/pingcap/followers","following_url":"https://api.github.com/users/pingcap/following{/other_user}","gists_url":"https://api.github.com/users/pingcap/gists{/gist_id}","starred_url":"https://api.github.com/users/pingcap/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pingcap/subscriptions","organizations_url":"https://api.github.com/users/pingcap/orgs","repos_url":"https://api.github.com/users/pingcap/repos","events_url":"https://api.github.com/users/pingcap/events{/privacy}","received_events_url":"https://api.github.com/users/pingcap/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/pingcap/oasis","description":null,"fork":false,"url":"https://api.github.com/repos/pingcap/oasis","forks_url":"https://api.github.com/repos/pingcap/oasis/forks","keys_url":"https://api.github.com/repos/pingcap/oasis/keys{/key_id}","collaborators_url":"https://api.github.com/repos/pingcap/oasis/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/pingcap/oasis/teams","hooks_url":"https://api.github.com/repos/pingcap/oasis/hooks","issue_events_url":"https://api.github.com/repos/pingcap/oasis/issues/events{/number}","events_url":"https://api.github.com/repos/pingcap/oasis/events","assignees_url":"https://api.github.com/repos/pingcap/oasis/assignees{/user}","branches_url":"https://api.github.com/repos/pingcap/oasis/branches{/branch}","tags_url":"https://api.github.com/repos/pingcap/oasis/tags","blobs_url":"https://api.github.com/repos/pingcap/oasis/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/pingcap/oasis/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/pingcap/oasis/git/refs{/sha}","trees_url":"https://api.github.com/repos/pingcap/oasis/git/trees{/sha}","statuses_url":"https://api.github.com/repos/pingcap/oasis/statuses/{sha}","languages_url":"https://api.github.com/repos/pingcap/oasis/languages","stargazers_url":"https://api.github.com/repos/pingcap/oasis/stargazers","contributors_url":"https://api.github.com/repos/pingcap/oasis/contributors","subscribers_url":"https://api.github.com/repos/pingcap/oasis/subscribers","subscription_url":"https://api.github.com/repos/pingcap/oasis/subscription","commits_url":"https://api.github.com/repos/pingcap/oasis/commits{/sha}","git_commits_url":"https://api.github.com/repos/pingcap/oasis/git/commits{/sha}","comments_url":"https://api.github.com/repos/pingcap/oasis/comments{/number}","issue_comment_url":"https://api.github.com/repos/pingcap/oasis/issues/comments{/number}","contents_url":"https://api.github.com/repos/pingcap/oasis/contents/{+path}","compare_url":"https://api.github.com/repos/pingcap/oasis/compare/{base}...{head}","merges_url":"https://api.github.com/repos/pingcap/oasis/merges","archive_url":"https://api.github.com/repos/pingcap/oasis/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/pingcap/oasis/downloads","issues_url":"https://api.github.com/repos/pingcap/oasis/issues{/number}","pulls_url":"https://api.github.com/repos/pingcap/oasis/pulls{/number}","milestones_url":"https://api.github.com/repos/pingcap/oasis/milestones{/number}","notifications_url":"https://api.github.com/repos/pingcap/oasis/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/pingcap/oasis/labels{/name}","releases_url":"https://api.github.com/repos/pingcap/oasis/releases{/id}","deployments_url":"https://api.github.com/repos/pingcap/oasis/deployments","created_at":"2018-04-02T06:54:14Z","updated_at":"2018-04-11T05:44:50Z","pushed_at":"2018-04-12T02:15:16Z","git_url":"git://github.com/pingcap/oasis.git","ssh_url":"git@github.com:pingcap/oasis.git","clone_url":"https://github.com/pingcap/oasis.git","svn_url":"https://github.com/pingcap/oasis","homepage":null,"size":48,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":2,"license":null,"forks":0,"open_issues":2,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/pingcap/oasis/pulls/4"},"html":{"href":"https://github.com/pingcap/oasis/pull/4"},"issue":{"href":"https://api.github.com/repos/pingcap/oasis/issues/4"},"comments":{"href":"https://api.github.com/repos/pingcap/oasis/issues/4/comments"},"review_comments":{"href":"https://api.github.com/repos/pingcap/oasis/pulls/4/comments"},"review_comment":{"href":"https://api.github.com/repos/pingcap/oasis/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/pingcap/oasis/pulls/4/commits"},"statuses":{"href":"https://api.github.com/repos/pingcap/oasis/statuses/13b70d12e584673e8a4acbed7d8b85095e22f805"}},"author_association":"MEMBER"}}
{"actor":{"display_login":"foreverbell"}}