{"action":"created","comment":{"url":"https://api.github.com/repos/managedbyq/mbq.atomiq/pulls/comments/220253592","pull_request_review_id":158618880,"id":220253592,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIyMDI1MzU5Mg==","diff_hunk":"@@ -29,23 +31,66 @@ def __init__(self, *args, **kwargs):\n         signal.signal(signal.SIGTERM, self.signal_handler.handle_signal)\n         signal.signal(signal.SIGQUIT, self.signal_handler.handle_signal)\n \n+        self.queues = {\n+            constants.QueueType.SNS: {\n+                'model': models.SNSTask,\n+                'consumer_class': consumers.SNSConsumer,\n+            },\n+            constants.QueueType.SQS: {\n+                'model': models.SQSTask,\n+                'consumer_class': consumers.SQSConsumer,\n+            },\n+            constants.QueueType.CELERY: {\n+                'model': models.CeleryTask,\n+                'consumer_class': consumers.CeleryConsumer,\n+            },\n+        }\n+\n     def add_arguments(self, parser):\n-        parser.add_argument('--queue', required=True)\n+        queue_type_choices = [c[0] for c in constants.QueueType.CHOICES]\n+        parser.add_argument('--queue', required=True, choices=queue_type_choices)\n         parser.add_argument('--celery-app', required=False)\n \n+    @utils.debounce(minutes=15)\n+    def cleanup_old_tasks():\n+        days_to_keep_old_tasks = constants.DEFAULT_DAYS_TO_KEEP_OLD_TASKS\n+        time_to_delete_before = arrow.utcnow().shift(days=-days_to_keep_old_tasks)\n+\n+        model = self.queues[options['queue']]['model']\n+        model.objects.filter(\n+            state__in=[constants.TaskStates.SUCCEEDED, constants.TaskStates.DELETED],\n+            created_at__lt=time_to_delete_before.datetime,\n+        ).delete()\n+\n+    @utils.debounce(seconds=15)\n+    def collect_metrics(self):\n+        queue_type = options['queue']\n+        model = self.queues[queue_type]['model']\n+\n+        state_counts = collections.Counter(model.objects.values_list('state', flat=True))\n+        for state, count in state_counts.items():\n+            _collector.gauge(\n+                'state_total',\n+                count,\n+                tags={'state': state, 'queue_type': queue_type},\n+            )\n+\n     def handle(self, *args, **options):\n         try:\n-            queue_name = options['queue']\n-            if queue_name == 'sns':\n-                consumer = consumers.SNSConsumer()\n-            if queue_name == 'sqs':\n-                consumer = consumers.SQSConsumer()\n-            if queue_name == 'celery':\n-                consumer = consumers.CeleryConsumer(celery_app=options['celery_app'])\n+            queue_type = options['queue']","path":"mbq/atomiq/management/commands/atomic_run_consumer.py","position":72,"original_position":72,"commit_id":"aecdffcea925aa15a3205186ab1d7526e3762d2a","original_commit_id":"aecdffcea925aa15a3205186ab1d7526e3762d2a","user":{"login":"Pastromhaug","id":7873986,"node_id":"MDQ6VXNlcjc4NzM5ODY=","avatar_url":"https://avatars0.githubusercontent.com/u/7873986?v=4","gravatar_id":"","url":"https://api.github.com/users/Pastromhaug","html_url":"https://github.com/Pastromhaug","followers_url":"https://api.github.com/users/Pastromhaug/followers","following_url":"https://api.github.com/users/Pastromhaug/following{/other_user}","gists_url":"https://api.github.com/users/Pastromhaug/gists{/gist_id}","starred_url":"https://api.github.com/users/Pastromhaug/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Pastromhaug/subscriptions","organizations_url":"https://api.github.com/users/Pastromhaug/orgs","repos_url":"https://api.github.com/users/Pastromhaug/repos","events_url":"https://api.github.com/users/Pastromhaug/events{/privacy}","received_events_url":"https://api.github.com/users/Pastromhaug/received_events","type":"User","site_admin":false},"body":"Nit: We can use this decorator instead of indenting the whole handle function in a try/except:\r\n(just added this in invoicing yesterday)\r\n```\r\ndef send_errors_to_rollbar(func):\r\n\r\n    def wrapper(*args, **kwargs):\r\n        try:\r\n            func(*args, **kwargs)\r\n        except Exception:\r\n            rollbar.report_exc_info()\r\n            raise\r\n\r\n    return wrapper\r\n```\r\n\r\n```\r\n@send_errors_to_rollbar\r\ndef handle(...):\r\n```","created_at":"2018-09-25T15:59:09Z","updated_at":"2018-09-25T15:59:09Z","html_url":"https://github.com/managedbyq/mbq.atomiq/pull/13#discussion_r220253592","pull_request_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/pulls/13","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/managedbyq/mbq.atomiq/pulls/comments/220253592"},"html":{"href":"https://github.com/managedbyq/mbq.atomiq/pull/13#discussion_r220253592"},"pull_request":{"href":"https://api.github.com/repos/managedbyq/mbq.atomiq/pulls/13"}}},"pull_request":{"url":"https://api.github.com/repos/managedbyq/mbq.atomiq/pulls/13","id":217784512,"node_id":"MDExOlB1bGxSZXF1ZXN0MjE3Nzg0NTEy","html_url":"https://github.com/managedbyq/mbq.atomiq/pull/13","diff_url":"https://github.com/managedbyq/mbq.atomiq/pull/13.diff","patch_url":"https://github.com/managedbyq/mbq.atomiq/pull/13.patch","issue_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/issues/13","number":13,"state":"open","locked":false,"title":"collect metrics","user":{"login":"sashako","id":1760792,"node_id":"MDQ6VXNlcjE3NjA3OTI=","avatar_url":"https://avatars2.githubusercontent.com/u/1760792?v=4","gravatar_id":"","url":"https://api.github.com/users/sashako","html_url":"https://github.com/sashako","followers_url":"https://api.github.com/users/sashako/followers","following_url":"https://api.github.com/users/sashako/following{/other_user}","gists_url":"https://api.github.com/users/sashako/gists{/gist_id}","starred_url":"https://api.github.com/users/sashako/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sashako/subscriptions","organizations_url":"https://api.github.com/users/sashako/orgs","repos_url":"https://api.github.com/users/sashako/repos","events_url":"https://api.github.com/users/sashako/events{/privacy}","received_events_url":"https://api.github.com/users/sashako/received_events","type":"User","site_admin":false},"body":"should be a separate cron, doesn't need to run as frequently as each run of the consumer","created_at":"2018-09-24T20:51:28Z","updated_at":"2018-09-25T15:59:09Z","closed_at":null,"merged_at":null,"merge_commit_sha":"436bc2e7edb32736614fe7b751c4c5ba5c88ff29","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/pulls/13/commits","review_comments_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/pulls/13/comments","review_comment_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/pulls/comments{/number}","comments_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/issues/13/comments","statuses_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/statuses/aecdffcea925aa15a3205186ab1d7526e3762d2a","head":{"label":"managedbyq:separate-collect-metrics","ref":"separate-collect-metrics","sha":"aecdffcea925aa15a3205186ab1d7526e3762d2a","user":{"login":"managedbyq","id":11465985,"node_id":"MDEyOk9yZ2FuaXphdGlvbjExNDY1OTg1","avatar_url":"https://avatars1.githubusercontent.com/u/11465985?v=4","gravatar_id":"","url":"https://api.github.com/users/managedbyq","html_url":"https://github.com/managedbyq","followers_url":"https://api.github.com/users/managedbyq/followers","following_url":"https://api.github.com/users/managedbyq/following{/other_user}","gists_url":"https://api.github.com/users/managedbyq/gists{/gist_id}","starred_url":"https://api.github.com/users/managedbyq/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/managedbyq/subscriptions","organizations_url":"https://api.github.com/users/managedbyq/orgs","repos_url":"https://api.github.com/users/managedbyq/repos","events_url":"https://api.github.com/users/managedbyq/events{/privacy}","received_events_url":"https://api.github.com/users/managedbyq/received_events","type":"Organization","site_admin":false},"repo":{"id":149150825,"node_id":"MDEwOlJlcG9zaXRvcnkxNDkxNTA4MjU=","name":"mbq.atomiq","full_name":"managedbyq/mbq.atomiq","private":false,"owner":{"login":"managedbyq","id":11465985,"node_id":"MDEyOk9yZ2FuaXphdGlvbjExNDY1OTg1","avatar_url":"https://avatars1.githubusercontent.com/u/11465985?v=4","gravatar_id":"","url":"https://api.github.com/users/managedbyq","html_url":"https://github.com/managedbyq","followers_url":"https://api.github.com/users/managedbyq/followers","following_url":"https://api.github.com/users/managedbyq/following{/other_user}","gists_url":"https://api.github.com/users/managedbyq/gists{/gist_id}","starred_url":"https://api.github.com/users/managedbyq/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/managedbyq/subscriptions","organizations_url":"https://api.github.com/users/managedbyq/orgs","repos_url":"https://api.github.com/users/managedbyq/repos","events_url":"https://api.github.com/users/managedbyq/events{/privacy}","received_events_url":"https://api.github.com/users/managedbyq/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/managedbyq/mbq.atomiq","description":null,"fork":false,"url":"https://api.github.com/repos/managedbyq/mbq.atomiq","forks_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/forks","keys_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/keys{/key_id}","collaborators_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/teams","hooks_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/hooks","issue_events_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/issues/events{/number}","events_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/events","assignees_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/assignees{/user}","branches_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/branches{/branch}","tags_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/tags","blobs_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/git/refs{/sha}","trees_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/git/trees{/sha}","statuses_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/statuses/{sha}","languages_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/languages","stargazers_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/stargazers","contributors_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/contributors","subscribers_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/subscribers","subscription_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/subscription","commits_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/commits{/sha}","git_commits_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/git/commits{/sha}","comments_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/comments{/number}","issue_comment_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/issues/comments{/number}","contents_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/contents/{+path}","compare_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/compare/{base}...{head}","merges_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/merges","archive_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/downloads","issues_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/issues{/number}","pulls_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/pulls{/number}","milestones_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/milestones{/number}","notifications_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/labels{/name}","releases_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/releases{/id}","deployments_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/deployments","created_at":"2018-09-17T15:56:48Z","updated_at":"2018-09-25T15:06:29Z","pushed_at":"2018-09-25T15:21:11Z","git_url":"git://github.com/managedbyq/mbq.atomiq.git","ssh_url":"git@github.com:managedbyq/mbq.atomiq.git","clone_url":"https://github.com/managedbyq/mbq.atomiq.git","svn_url":"https://github.com/managedbyq/mbq.atomiq","homepage":null,"size":69,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":1,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"managedbyq:master","ref":"master","sha":"c4f3aad4289a3b1eda550c8467f23aebfa4e1a4e","user":{"login":"managedbyq","id":11465985,"node_id":"MDEyOk9yZ2FuaXphdGlvbjExNDY1OTg1","avatar_url":"https://avatars1.githubusercontent.com/u/11465985?v=4","gravatar_id":"","url":"https://api.github.com/users/managedbyq","html_url":"https://github.com/managedbyq","followers_url":"https://api.github.com/users/managedbyq/followers","following_url":"https://api.github.com/users/managedbyq/following{/other_user}","gists_url":"https://api.github.com/users/managedbyq/gists{/gist_id}","starred_url":"https://api.github.com/users/managedbyq/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/managedbyq/subscriptions","organizations_url":"https://api.github.com/users/managedbyq/orgs","repos_url":"https://api.github.com/users/managedbyq/repos","events_url":"https://api.github.com/users/managedbyq/events{/privacy}","received_events_url":"https://api.github.com/users/managedbyq/received_events","type":"Organization","site_admin":false},"repo":{"id":149150825,"node_id":"MDEwOlJlcG9zaXRvcnkxNDkxNTA4MjU=","name":"mbq.atomiq","full_name":"managedbyq/mbq.atomiq","private":false,"owner":{"login":"managedbyq","id":11465985,"node_id":"MDEyOk9yZ2FuaXphdGlvbjExNDY1OTg1","avatar_url":"https://avatars1.githubusercontent.com/u/11465985?v=4","gravatar_id":"","url":"https://api.github.com/users/managedbyq","html_url":"https://github.com/managedbyq","followers_url":"https://api.github.com/users/managedbyq/followers","following_url":"https://api.github.com/users/managedbyq/following{/other_user}","gists_url":"https://api.github.com/users/managedbyq/gists{/gist_id}","starred_url":"https://api.github.com/users/managedbyq/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/managedbyq/subscriptions","organizations_url":"https://api.github.com/users/managedbyq/orgs","repos_url":"https://api.github.com/users/managedbyq/repos","events_url":"https://api.github.com/users/managedbyq/events{/privacy}","received_events_url":"https://api.github.com/users/managedbyq/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/managedbyq/mbq.atomiq","description":null,"fork":false,"url":"https://api.github.com/repos/managedbyq/mbq.atomiq","forks_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/forks","keys_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/keys{/key_id}","collaborators_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/teams","hooks_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/hooks","issue_events_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/issues/events{/number}","events_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/events","assignees_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/assignees{/user}","branches_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/branches{/branch}","tags_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/tags","blobs_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/git/refs{/sha}","trees_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/git/trees{/sha}","statuses_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/statuses/{sha}","languages_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/languages","stargazers_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/stargazers","contributors_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/contributors","subscribers_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/subscribers","subscription_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/subscription","commits_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/commits{/sha}","git_commits_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/git/commits{/sha}","comments_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/comments{/number}","issue_comment_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/issues/comments{/number}","contents_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/contents/{+path}","compare_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/compare/{base}...{head}","merges_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/merges","archive_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/downloads","issues_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/issues{/number}","pulls_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/pulls{/number}","milestones_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/milestones{/number}","notifications_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/labels{/name}","releases_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/releases{/id}","deployments_url":"https://api.github.com/repos/managedbyq/mbq.atomiq/deployments","created_at":"2018-09-17T15:56:48Z","updated_at":"2018-09-25T15:06:29Z","pushed_at":"2018-09-25T15:21:11Z","git_url":"git://github.com/managedbyq/mbq.atomiq.git","ssh_url":"git@github.com:managedbyq/mbq.atomiq.git","clone_url":"https://github.com/managedbyq/mbq.atomiq.git","svn_url":"https://github.com/managedbyq/mbq.atomiq","homepage":null,"size":69,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":1,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/managedbyq/mbq.atomiq/pulls/13"},"html":{"href":"https://github.com/managedbyq/mbq.atomiq/pull/13"},"issue":{"href":"https://api.github.com/repos/managedbyq/mbq.atomiq/issues/13"},"comments":{"href":"https://api.github.com/repos/managedbyq/mbq.atomiq/issues/13/comments"},"review_comments":{"href":"https://api.github.com/repos/managedbyq/mbq.atomiq/pulls/13/comments"},"review_comment":{"href":"https://api.github.com/repos/managedbyq/mbq.atomiq/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/managedbyq/mbq.atomiq/pulls/13/commits"},"statuses":{"href":"https://api.github.com/repos/managedbyq/mbq.atomiq/statuses/aecdffcea925aa15a3205186ab1d7526e3762d2a"}},"author_association":"MEMBER"}}
{"actor":{"display_login":"Pastromhaug"}}