{"action":"created","comment":{"url":"https://api.github.com/repos/yandex/mastermind/pulls/comments/91962739","pull_request_review_id":12471734,"id":91962739,"diff_hunk":"@@ -0,0 +1,176 @@\n+from logging import getLogger\n+from yql.api.v1.client import YqlClient\n+import time\n+import datetime\n+\n+\n+logger = getLogger('mm.planner')\n+\n+\n+class YqlWrapper(object):\n+    def __init__(self, cluster, token, attempts, delay):\n+        self._cluster = cluster\n+        self._token = token\n+        self._attempts = attempts\n+        self._delay = delay\n+        self.aggregate_query = \"\"\"\n+        SELECT couple_id\n+        FROM\n+            (SELECT\n+                        couple_id,\n+                        SUM(expired_size) AS sum_expired_size\n+                    FROM [{table}]\n+                    WHERE expiration_date <= {timestamp}\n+                    GROUP BY couple_id)\n+            WHERE  sum_expired_size >= {trigger};\n+        \"\"\"\n+\n+        # XXX: maybe something more optimal could be done? How the db is organized?\n+        # XXX: tounixdate -> arithmetic ops\n+        # XXX: use HAVING ?\n+        self.preaggregate_query = \"\"\"\n+$dateconv = @@\n+import time\n+import datetime\n+def tounixdate(exp_time):\n+    return int(time.mktime((datetime.date.fromtimestamp(exp_time) + datetime.timedelta(days=1)).timetuple()))\n+@@;\n+\n+$dateconvert = Python::tounixdate(\"(Uint64?)->Uint64\", $dateconv);\n+\n+INSERT INTO [{agr_table}]\n+SELECT\n+    couple_id,\n+    expiration_date,\n+    namespace,\n+    \"upload\" AS operation,\n+    SUM(CAST(object_size AS Int64)) AS expired_size,\n+    \"{date_iso}\" AS source_table,\n+    {timestamp} AS timestamp\n+FROM [{main_table}]\n+WHERE op=\"upload\"\n+GROUP BY\n+    couple_id,\n+    namespace,\n+    $dateconvert(CAST(expire_at AS Uint64)) AS expiration_date;\n+\n+INSERT INTO [{agr_table}]\n+SELECT\n+    couple_id,\n+    expiration_date,\n+    namespace,\n+    \"delete\" AS operation,\n+    -1*SUM(CAST(object_size AS Int64)) AS expired_size,\n+    \"{date_iso}\" AS source_table,\n+    {timestamp} AS timestamp\n+FROM [{main_table}]\n+WHERE op=\"delete\" AND expire_at IS NOT NULL\n+GROUP BY\n+    couple_id,\n+    namespace,\n+    $dateconvert(CAST(expire_at AS Uint64)) AS expiration_date;\n+\"\"\"\n+\n+    def send_request(self, query, timeout=None):\n+        \"\"\"\n+        Send request to YQL: ping verification + attempts retry\n+        @return result dict on success, excepts otherwise\n+        \"\"\"\n+        logger.debug('Send YQL request {}'.format(query))\n+        result = None\n+\n+        with YqlClient(db=self._cluster, token=self._token) as yql:\n+            try:\n+                if not yql.ping():\n+                    raise IOError(\"YQL ping failed {} {}\".format(self._cluster, self._token))\n+            except:\n+                logger.exception(\"YQL ping excepted {}{}\".format(self._cluster, self._token))\n+                raise\n+\n+            start_time = time.time()\n+\n+            for attempt in xrange(self._attempts):\n+                try:\n+                    request = yql.query(query)\n+                    request.run()\n+\n+                    logger.info(\"YQL query is running with id {}\".format(request.operation_id))\n+                    result = request.results\n+                    end_time = time.time()\n+                    if result.is_success:\n+                        logger.info(\"YQL query succeeded and took {}\".format(end_time-start_time))\n+                        if timeout and (end_time - start_time > timeout):\n+                            raise AssertionError(\"YQL query({}..{}) > timeout {}\".format(start_time, end_time, timeout))\n+                        return result\n+\n+                    logger.error(\"YQL failed #{}: {}[{}] ({},{!s})\".format(attempt, result.status,\n+                                 request.status_code, request.explain(), request.exc_info))\n+                    for error in result.errors:\n+                        logger.error(\"YQL query result error {!s}\".format(error))\n+\n+                    time.sleep(self._delay)\n+                except:\n+                    logger.exception(\"YQL request excepted\")\n+                    raise\n+\n+        raise AssertionError(\"YQL request attempts has exhausted {} {}\".format(self._attempts, query))\n+\n+\n+    def request_expired_stat(self, aggregate_table, expired_threshold, timeout=None):\n+        \"\"\"\n+        Send request to aggregate table to find volume of expired space in couples\n+        :param aggregate_table: the name of aggregation table\n+        :param expired_threshold: a couples with expired volume above param are suspected","path":"src/cocaine-app/yt_worker.py","position":null,"original_position":123,"commit_id":"63dda050e170bb80778d39c9eec3aabd8f24a186","original_commit_id":"e8a3c836b3d2e753cccd6326c8c4cb9f48cec5ba","user":{"login":"annamel","id":11502005,"avatar_url":"https://avatars.githubusercontent.com/u/11502005?v=3","gravatar_id":"","url":"https://api.github.com/users/annamel","html_url":"https://github.com/annamel","followers_url":"https://api.github.com/users/annamel/followers","following_url":"https://api.github.com/users/annamel/following{/other_user}","gists_url":"https://api.github.com/users/annamel/gists{/gist_id}","starred_url":"https://api.github.com/users/annamel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/annamel/subscriptions","organizations_url":"https://api.github.com/users/annamel/orgs","repos_url":"https://api.github.com/users/annamel/repos","events_url":"https://api.github.com/users/annamel/events{/privacy}","received_events_url":"https://api.github.com/users/annamel/received_events","type":"User","site_admin":false},"body":"Done in https://github.com/yandex/mastermind/pull/311/commits/4732106cd00d7b48fb9a6db30b7cff4aef8db6df","created_at":"2016-12-12T15:01:25Z","updated_at":"2016-12-12T15:01:25Z","html_url":"https://github.com/yandex/mastermind/pull/311#discussion_r91962739","pull_request_url":"https://api.github.com/repos/yandex/mastermind/pulls/311","_links":{"self":{"href":"https://api.github.com/repos/yandex/mastermind/pulls/comments/91962739"},"html":{"href":"https://github.com/yandex/mastermind/pull/311#discussion_r91962739"},"pull_request":{"href":"https://api.github.com/repos/yandex/mastermind/pulls/311"}}},"pull_request":{"url":"https://api.github.com/repos/yandex/mastermind/pulls/311","id":94003498,"html_url":"https://github.com/yandex/mastermind/pull/311","diff_url":"https://github.com/yandex/mastermind/pull/311.diff","patch_url":"https://github.com/yandex/mastermind/pull/311.patch","issue_url":"https://api.github.com/repos/yandex/mastermind/issues/311","number":311,"state":"open","locked":false,"title":"TTL cleanup scheduler. v0.1","user":{"login":"annamel","id":11502005,"avatar_url":"https://avatars.githubusercontent.com/u/11502005?v=3","gravatar_id":"","url":"https://api.github.com/users/annamel","html_url":"https://github.com/annamel","followers_url":"https://api.github.com/users/annamel/followers","following_url":"https://api.github.com/users/annamel/following{/other_user}","gists_url":"https://api.github.com/users/annamel/gists{/gist_id}","starred_url":"https://api.github.com/users/annamel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/annamel/subscriptions","organizations_url":"https://api.github.com/users/annamel/orgs","repos_url":"https://api.github.com/users/annamel/repos","events_url":"https://api.github.com/users/annamel/events{/privacy}","received_events_url":"https://api.github.com/users/annamel/received_events","type":"User","site_admin":false},"body":"The first approximation of ttl cleanup scheduler. It starts periodically, aggregates data from stat_box of the previous day into YT sub-table. And then based on YT sub-table information and configuration settings, makes a decision whether it worth running mds_cleanup service.\r\n\r\nThe current solution doesn't analyze the history of runs. And it doesn't take into account inter-crossing between ttl_cleanup on the same HDD. These features are coming","created_at":"2016-11-16T16:43:42Z","updated_at":"2016-12-12T15:01:25Z","closed_at":null,"merged_at":null,"merge_commit_sha":"4d775044ed4a10d221aa3e0b036e8063fc69a0ab","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/yandex/mastermind/pulls/311/commits","review_comments_url":"https://api.github.com/repos/yandex/mastermind/pulls/311/comments","review_comment_url":"https://api.github.com/repos/yandex/mastermind/pulls/comments{/number}","comments_url":"https://api.github.com/repos/yandex/mastermind/issues/311/comments","statuses_url":"https://api.github.com/repos/yandex/mastermind/statuses/63dda050e170bb80778d39c9eec3aabd8f24a186","head":{"label":"annamel:ttl_cleanup_sched","ref":"ttl_cleanup_sched","sha":"63dda050e170bb80778d39c9eec3aabd8f24a186","user":{"login":"annamel","id":11502005,"avatar_url":"https://avatars.githubusercontent.com/u/11502005?v=3","gravatar_id":"","url":"https://api.github.com/users/annamel","html_url":"https://github.com/annamel","followers_url":"https://api.github.com/users/annamel/followers","following_url":"https://api.github.com/users/annamel/following{/other_user}","gists_url":"https://api.github.com/users/annamel/gists{/gist_id}","starred_url":"https://api.github.com/users/annamel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/annamel/subscriptions","organizations_url":"https://api.github.com/users/annamel/orgs","repos_url":"https://api.github.com/users/annamel/repos","events_url":"https://api.github.com/users/annamel/events{/privacy}","received_events_url":"https://api.github.com/users/annamel/received_events","type":"User","site_admin":false},"repo":{"id":66254618,"name":"mastermind","full_name":"annamel/mastermind","owner":{"login":"annamel","id":11502005,"avatar_url":"https://avatars.githubusercontent.com/u/11502005?v=3","gravatar_id":"","url":"https://api.github.com/users/annamel","html_url":"https://github.com/annamel","followers_url":"https://api.github.com/users/annamel/followers","following_url":"https://api.github.com/users/annamel/following{/other_user}","gists_url":"https://api.github.com/users/annamel/gists{/gist_id}","starred_url":"https://api.github.com/users/annamel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/annamel/subscriptions","organizations_url":"https://api.github.com/users/annamel/orgs","repos_url":"https://api.github.com/users/annamel/repos","events_url":"https://api.github.com/users/annamel/events{/privacy}","received_events_url":"https://api.github.com/users/annamel/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/annamel/mastermind","description":"Smart control for a big storage","fork":true,"url":"https://api.github.com/repos/annamel/mastermind","forks_url":"https://api.github.com/repos/annamel/mastermind/forks","keys_url":"https://api.github.com/repos/annamel/mastermind/keys{/key_id}","collaborators_url":"https://api.github.com/repos/annamel/mastermind/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/annamel/mastermind/teams","hooks_url":"https://api.github.com/repos/annamel/mastermind/hooks","issue_events_url":"https://api.github.com/repos/annamel/mastermind/issues/events{/number}","events_url":"https://api.github.com/repos/annamel/mastermind/events","assignees_url":"https://api.github.com/repos/annamel/mastermind/assignees{/user}","branches_url":"https://api.github.com/repos/annamel/mastermind/branches{/branch}","tags_url":"https://api.github.com/repos/annamel/mastermind/tags","blobs_url":"https://api.github.com/repos/annamel/mastermind/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/annamel/mastermind/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/annamel/mastermind/git/refs{/sha}","trees_url":"https://api.github.com/repos/annamel/mastermind/git/trees{/sha}","statuses_url":"https://api.github.com/repos/annamel/mastermind/statuses/{sha}","languages_url":"https://api.github.com/repos/annamel/mastermind/languages","stargazers_url":"https://api.github.com/repos/annamel/mastermind/stargazers","contributors_url":"https://api.github.com/repos/annamel/mastermind/contributors","subscribers_url":"https://api.github.com/repos/annamel/mastermind/subscribers","subscription_url":"https://api.github.com/repos/annamel/mastermind/subscription","commits_url":"https://api.github.com/repos/annamel/mastermind/commits{/sha}","git_commits_url":"https://api.github.com/repos/annamel/mastermind/git/commits{/sha}","comments_url":"https://api.github.com/repos/annamel/mastermind/comments{/number}","issue_comment_url":"https://api.github.com/repos/annamel/mastermind/issues/comments{/number}","contents_url":"https://api.github.com/repos/annamel/mastermind/contents/{+path}","compare_url":"https://api.github.com/repos/annamel/mastermind/compare/{base}...{head}","merges_url":"https://api.github.com/repos/annamel/mastermind/merges","archive_url":"https://api.github.com/repos/annamel/mastermind/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/annamel/mastermind/downloads","issues_url":"https://api.github.com/repos/annamel/mastermind/issues{/number}","pulls_url":"https://api.github.com/repos/annamel/mastermind/pulls{/number}","milestones_url":"https://api.github.com/repos/annamel/mastermind/milestones{/number}","notifications_url":"https://api.github.com/repos/annamel/mastermind/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/annamel/mastermind/labels{/name}","releases_url":"https://api.github.com/repos/annamel/mastermind/releases{/id}","deployments_url":"https://api.github.com/repos/annamel/mastermind/deployments","created_at":"2016-08-22T08:38:42Z","updated_at":"2016-08-22T08:38:45Z","pushed_at":"2016-12-12T14:51:07Z","git_url":"git://github.com/annamel/mastermind.git","ssh_url":"git@github.com:annamel/mastermind.git","clone_url":"https://github.com/annamel/mastermind.git","svn_url":"https://github.com/annamel/mastermind","homepage":"","size":7911,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"yandex:release-2.28","ref":"release-2.28","sha":"8b79d9fca03524261e195f12a767533c45be47d0","user":{"login":"yandex","id":7409213,"avatar_url":"https://avatars.githubusercontent.com/u/7409213?v=3","gravatar_id":"","url":"https://api.github.com/users/yandex","html_url":"https://github.com/yandex","followers_url":"https://api.github.com/users/yandex/followers","following_url":"https://api.github.com/users/yandex/following{/other_user}","gists_url":"https://api.github.com/users/yandex/gists{/gist_id}","starred_url":"https://api.github.com/users/yandex/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yandex/subscriptions","organizations_url":"https://api.github.com/users/yandex/orgs","repos_url":"https://api.github.com/users/yandex/repos","events_url":"https://api.github.com/users/yandex/events{/privacy}","received_events_url":"https://api.github.com/users/yandex/received_events","type":"Organization","site_admin":false},"repo":{"id":23435885,"name":"mastermind","full_name":"yandex/mastermind","owner":{"login":"yandex","id":7409213,"avatar_url":"https://avatars.githubusercontent.com/u/7409213?v=3","gravatar_id":"","url":"https://api.github.com/users/yandex","html_url":"https://github.com/yandex","followers_url":"https://api.github.com/users/yandex/followers","following_url":"https://api.github.com/users/yandex/following{/other_user}","gists_url":"https://api.github.com/users/yandex/gists{/gist_id}","starred_url":"https://api.github.com/users/yandex/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yandex/subscriptions","organizations_url":"https://api.github.com/users/yandex/orgs","repos_url":"https://api.github.com/users/yandex/repos","events_url":"https://api.github.com/users/yandex/events{/privacy}","received_events_url":"https://api.github.com/users/yandex/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/yandex/mastermind","description":"Smart control for a big storage","fork":false,"url":"https://api.github.com/repos/yandex/mastermind","forks_url":"https://api.github.com/repos/yandex/mastermind/forks","keys_url":"https://api.github.com/repos/yandex/mastermind/keys{/key_id}","collaborators_url":"https://api.github.com/repos/yandex/mastermind/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/yandex/mastermind/teams","hooks_url":"https://api.github.com/repos/yandex/mastermind/hooks","issue_events_url":"https://api.github.com/repos/yandex/mastermind/issues/events{/number}","events_url":"https://api.github.com/repos/yandex/mastermind/events","assignees_url":"https://api.github.com/repos/yandex/mastermind/assignees{/user}","branches_url":"https://api.github.com/repos/yandex/mastermind/branches{/branch}","tags_url":"https://api.github.com/repos/yandex/mastermind/tags","blobs_url":"https://api.github.com/repos/yandex/mastermind/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/yandex/mastermind/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/yandex/mastermind/git/refs{/sha}","trees_url":"https://api.github.com/repos/yandex/mastermind/git/trees{/sha}","statuses_url":"https://api.github.com/repos/yandex/mastermind/statuses/{sha}","languages_url":"https://api.github.com/repos/yandex/mastermind/languages","stargazers_url":"https://api.github.com/repos/yandex/mastermind/stargazers","contributors_url":"https://api.github.com/repos/yandex/mastermind/contributors","subscribers_url":"https://api.github.com/repos/yandex/mastermind/subscribers","subscription_url":"https://api.github.com/repos/yandex/mastermind/subscription","commits_url":"https://api.github.com/repos/yandex/mastermind/commits{/sha}","git_commits_url":"https://api.github.com/repos/yandex/mastermind/git/commits{/sha}","comments_url":"https://api.github.com/repos/yandex/mastermind/comments{/number}","issue_comment_url":"https://api.github.com/repos/yandex/mastermind/issues/comments{/number}","contents_url":"https://api.github.com/repos/yandex/mastermind/contents/{+path}","compare_url":"https://api.github.com/repos/yandex/mastermind/compare/{base}...{head}","merges_url":"https://api.github.com/repos/yandex/mastermind/merges","archive_url":"https://api.github.com/repos/yandex/mastermind/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/yandex/mastermind/downloads","issues_url":"https://api.github.com/repos/yandex/mastermind/issues{/number}","pulls_url":"https://api.github.com/repos/yandex/mastermind/pulls{/number}","milestones_url":"https://api.github.com/repos/yandex/mastermind/milestones{/number}","notifications_url":"https://api.github.com/repos/yandex/mastermind/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/yandex/mastermind/labels{/name}","releases_url":"https://api.github.com/repos/yandex/mastermind/releases{/id}","deployments_url":"https://api.github.com/repos/yandex/mastermind/deployments","created_at":"2014-08-28T16:57:05Z","updated_at":"2016-11-21T01:57:19Z","pushed_at":"2016-12-12T14:51:08Z","git_url":"git://github.com/yandex/mastermind.git","ssh_url":"git@github.com:yandex/mastermind.git","clone_url":"https://github.com/yandex/mastermind.git","svn_url":"https://github.com/yandex/mastermind","homepage":"","size":8557,"stargazers_count":10,"watchers_count":10,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":15,"mirror_url":null,"open_issues_count":26,"forks":15,"open_issues":26,"watchers":10,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/yandex/mastermind/pulls/311"},"html":{"href":"https://github.com/yandex/mastermind/pull/311"},"issue":{"href":"https://api.github.com/repos/yandex/mastermind/issues/311"},"comments":{"href":"https://api.github.com/repos/yandex/mastermind/issues/311/comments"},"review_comments":{"href":"https://api.github.com/repos/yandex/mastermind/pulls/311/comments"},"review_comment":{"href":"https://api.github.com/repos/yandex/mastermind/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/yandex/mastermind/pulls/311/commits"},"statuses":{"href":"https://api.github.com/repos/yandex/mastermind/statuses/63dda050e170bb80778d39c9eec3aabd8f24a186"}}}}
{"actor":{"display_login":"annamel"}}