{"action":"created","comment":{"url":"https://api.github.com/repos/mozilla/socorro/pulls/comments/57742255","id":57742255,"diff_hunk":"@@ -51,6 +84,124 @@ def create_correlations_index(self, es_index):\n             es_settings = json.loads(settings_json)\n \n             index_creator = self.config.index_creator(self.config)\n+            # This is resilient to being called repeatedly even\n+            # when the index already exists.\n             index_creator.create_index(es_index, es_settings)\n-\n             self.indices_cache.add(es_index)\n+\n+    def _prefix_to_datetime_date(self, prefix):\n+        yy = int(prefix[:4])\n+        mm = int(prefix[4:6])\n+        dd = int(prefix[6:8])\n+        return datetime.date(yy, mm, dd)\n+\n+    @staticmethod\n+    def make_id(document):\n+        string_parts = []\n+        for key, value in sorted(document.items()):\n+            if isinstance(value, datetime.date):\n+                value = value.isoformat()\n+            if isinstance(value, int):\n+                value = str(value)\n+            if isinstance(value, basestring):\n+                string_parts.append(value)\n+        return hashlib.md5(\n+            (''.join(string_parts)).encode('utf-8')\n+        ).hexdigest()\n+\n+\n+class CoreCounts(Correlations):\n+\n+    def store(\n+        self,\n+        counts_summary_structure,\n+        prefix,\n+        name,\n+        key,\n+    ):\n+        date = self._prefix_to_datetime_date(prefix)\n+        index = self.get_index_for_date(date)\n+        self.create_correlations_index(index)\n+\n+        notes = counts_summary_structure['notes']\n+        product = key.split('_')[0]\n+        version = key.split('_')[1]","path":"socorro/external/es/correlations.py","position":113,"original_position":113,"commit_id":"8388dab64f56aecb90b034d1ae566b9a545da476","original_commit_id":"8388dab64f56aecb90b034d1ae566b9a545da476","user":{"login":"adngdb","id":328790,"avatar_url":"https://avatars.githubusercontent.com/u/328790?v=3","gravatar_id":"","url":"https://api.github.com/users/adngdb","html_url":"https://github.com/adngdb","followers_url":"https://api.github.com/users/adngdb/followers","following_url":"https://api.github.com/users/adngdb/following{/other_user}","gists_url":"https://api.github.com/users/adngdb/gists{/gist_id}","starred_url":"https://api.github.com/users/adngdb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/adngdb/subscriptions","organizations_url":"https://api.github.com/users/adngdb/orgs","repos_url":"https://api.github.com/users/adngdb/repos","events_url":"https://api.github.com/users/adngdb/events{/privacy}","received_events_url":"https://api.github.com/users/adngdb/received_events","type":"User","site_admin":false},"body":"Isn't this better: ``product, version = key.split('_', 1)``? ","created_at":"2016-03-29T15:21:42Z","updated_at":"2016-03-29T15:21:42Z","html_url":"https://github.com/mozilla/socorro/pull/3251#discussion_r57742255","pull_request_url":"https://api.github.com/repos/mozilla/socorro/pulls/3251","_links":{"self":{"href":"https://api.github.com/repos/mozilla/socorro/pulls/comments/57742255"},"html":{"href":"https://github.com/mozilla/socorro/pull/3251#discussion_r57742255"},"pull_request":{"href":"https://api.github.com/repos/mozilla/socorro/pulls/3251"}}},"pull_request":{"url":"https://api.github.com/repos/mozilla/socorro/pulls/3251","id":64184439,"html_url":"https://github.com/mozilla/socorro/pull/3251","diff_url":"https://github.com/mozilla/socorro/pull/3251.diff","patch_url":"https://github.com/mozilla/socorro/pull/3251.patch","issue_url":"https://api.github.com/repos/mozilla/socorro/issues/3251","number":3251,"state":"open","locked":false,"title":"Bug 1249359 store correlations in es","user":{"login":"peterbe","id":26739,"avatar_url":"https://avatars.githubusercontent.com/u/26739?v=3","gravatar_id":"","url":"https://api.github.com/users/peterbe","html_url":"https://github.com/peterbe","followers_url":"https://api.github.com/users/peterbe/followers","following_url":"https://api.github.com/users/peterbe/following{/other_user}","gists_url":"https://api.github.com/users/peterbe/gists{/gist_id}","starred_url":"https://api.github.com/users/peterbe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/peterbe/subscriptions","organizations_url":"https://api.github.com/users/peterbe/orgs","repos_url":"https://api.github.com/users/peterbe/repos","events_url":"https://api.github.com/users/peterbe/events{/privacy}","received_events_url":"https://api.github.com/users/peterbe/received_events","type":"User","site_admin":false},"body":"The way to run this is like this:\r\n\r\n```\r\nsocorro correlations  \\\r\n--resource.postgresql.database_port=5433 \\\r\n--resource.postgresql.database_name=breakpad \\\r\n--secrets.postgresql.database_username=breakpad_rw \\\r\n--secrets.postgresql.database_password=*********** \\\r\n--resource.boto.access_key=\"**********************\" \\\r\n--secrets.boto.secret_access_key=\"*********************\" \\\r\n--resource.boto.resource_class=socorro.external.boto.connection_context.RegionalS3ConnectionContext \\\r\n--resource.boto.bucket_name=\"org.allizom.crash-stats.rhelmer-test.crashes\" \\\r\n--resource.postgresql.transaction_executor_class=socorro.database.transaction_executor.TransactionExecutor \\\r\n--global.correlations.core.output_class=socorro.external.es.correlations.CoreCounts \\\r\n--global.correlations.interesting.output_class=socorro.external.es.correlations.InterestingModules \\\r\n--new_crash_source.elasticsearch.elasticsearch_urls=localhost:9222 \\\r\n--source.crashstorage_class=socorro.analysis.correlations.correlations_app.LocallyCachedBotoS3CrashStorage \\\r\n--producer_consumer.number_of_threads=5 \\\r\n--new_crash_source.cap=2000 \\\r\n$@\r\n```\r\n\r\nThat'll start 5 threads that jointly download 2000 processed crashes and then generates the summaries which it inserts into a local ES. \r\n\r\nNote: I have my ssh tunnel set to so that localhost:9222 is the stage ES. That's where I get a days worth of UUIDs from. ","created_at":"2016-03-25T17:15:57Z","updated_at":"2016-03-29T15:21:42Z","closed_at":null,"merged_at":null,"merge_commit_sha":"74e117c0bfc4cf9534cd323e93ee5320c7575915","assignee":{"login":"adngdb","id":328790,"avatar_url":"https://avatars.githubusercontent.com/u/328790?v=3","gravatar_id":"","url":"https://api.github.com/users/adngdb","html_url":"https://github.com/adngdb","followers_url":"https://api.github.com/users/adngdb/followers","following_url":"https://api.github.com/users/adngdb/following{/other_user}","gists_url":"https://api.github.com/users/adngdb/gists{/gist_id}","starred_url":"https://api.github.com/users/adngdb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/adngdb/subscriptions","organizations_url":"https://api.github.com/users/adngdb/orgs","repos_url":"https://api.github.com/users/adngdb/repos","events_url":"https://api.github.com/users/adngdb/events{/privacy}","received_events_url":"https://api.github.com/users/adngdb/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/mozilla/socorro/pulls/3251/commits","review_comments_url":"https://api.github.com/repos/mozilla/socorro/pulls/3251/comments","review_comment_url":"https://api.github.com/repos/mozilla/socorro/pulls/comments{/number}","comments_url":"https://api.github.com/repos/mozilla/socorro/issues/3251/comments","statuses_url":"https://api.github.com/repos/mozilla/socorro/statuses/8388dab64f56aecb90b034d1ae566b9a545da476","head":{"label":"peterbe:bug-1249359-store-correlations-in-es","ref":"bug-1249359-store-correlations-in-es","sha":"8388dab64f56aecb90b034d1ae566b9a545da476","user":{"login":"peterbe","id":26739,"avatar_url":"https://avatars.githubusercontent.com/u/26739?v=3","gravatar_id":"","url":"https://api.github.com/users/peterbe","html_url":"https://github.com/peterbe","followers_url":"https://api.github.com/users/peterbe/followers","following_url":"https://api.github.com/users/peterbe/following{/other_user}","gists_url":"https://api.github.com/users/peterbe/gists{/gist_id}","starred_url":"https://api.github.com/users/peterbe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/peterbe/subscriptions","organizations_url":"https://api.github.com/users/peterbe/orgs","repos_url":"https://api.github.com/users/peterbe/repos","events_url":"https://api.github.com/users/peterbe/events{/privacy}","received_events_url":"https://api.github.com/users/peterbe/received_events","type":"User","site_admin":false},"repo":{"id":2920665,"name":"socorro","full_name":"peterbe/socorro","owner":{"login":"peterbe","id":26739,"avatar_url":"https://avatars.githubusercontent.com/u/26739?v=3","gravatar_id":"","url":"https://api.github.com/users/peterbe","html_url":"https://github.com/peterbe","followers_url":"https://api.github.com/users/peterbe/followers","following_url":"https://api.github.com/users/peterbe/following{/other_user}","gists_url":"https://api.github.com/users/peterbe/gists{/gist_id}","starred_url":"https://api.github.com/users/peterbe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/peterbe/subscriptions","organizations_url":"https://api.github.com/users/peterbe/orgs","repos_url":"https://api.github.com/users/peterbe/repos","events_url":"https://api.github.com/users/peterbe/events{/privacy}","received_events_url":"https://api.github.com/users/peterbe/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/peterbe/socorro","description":"Socorro is a server to accept and process Breakpad crash reports. ","fork":true,"url":"https://api.github.com/repos/peterbe/socorro","forks_url":"https://api.github.com/repos/peterbe/socorro/forks","keys_url":"https://api.github.com/repos/peterbe/socorro/keys{/key_id}","collaborators_url":"https://api.github.com/repos/peterbe/socorro/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/peterbe/socorro/teams","hooks_url":"https://api.github.com/repos/peterbe/socorro/hooks","issue_events_url":"https://api.github.com/repos/peterbe/socorro/issues/events{/number}","events_url":"https://api.github.com/repos/peterbe/socorro/events","assignees_url":"https://api.github.com/repos/peterbe/socorro/assignees{/user}","branches_url":"https://api.github.com/repos/peterbe/socorro/branches{/branch}","tags_url":"https://api.github.com/repos/peterbe/socorro/tags","blobs_url":"https://api.github.com/repos/peterbe/socorro/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/peterbe/socorro/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/peterbe/socorro/git/refs{/sha}","trees_url":"https://api.github.com/repos/peterbe/socorro/git/trees{/sha}","statuses_url":"https://api.github.com/repos/peterbe/socorro/statuses/{sha}","languages_url":"https://api.github.com/repos/peterbe/socorro/languages","stargazers_url":"https://api.github.com/repos/peterbe/socorro/stargazers","contributors_url":"https://api.github.com/repos/peterbe/socorro/contributors","subscribers_url":"https://api.github.com/repos/peterbe/socorro/subscribers","subscription_url":"https://api.github.com/repos/peterbe/socorro/subscription","commits_url":"https://api.github.com/repos/peterbe/socorro/commits{/sha}","git_commits_url":"https://api.github.com/repos/peterbe/socorro/git/commits{/sha}","comments_url":"https://api.github.com/repos/peterbe/socorro/comments{/number}","issue_comment_url":"https://api.github.com/repos/peterbe/socorro/issues/comments{/number}","contents_url":"https://api.github.com/repos/peterbe/socorro/contents/{+path}","compare_url":"https://api.github.com/repos/peterbe/socorro/compare/{base}...{head}","merges_url":"https://api.github.com/repos/peterbe/socorro/merges","archive_url":"https://api.github.com/repos/peterbe/socorro/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/peterbe/socorro/downloads","issues_url":"https://api.github.com/repos/peterbe/socorro/issues{/number}","pulls_url":"https://api.github.com/repos/peterbe/socorro/pulls{/number}","milestones_url":"https://api.github.com/repos/peterbe/socorro/milestones{/number}","notifications_url":"https://api.github.com/repos/peterbe/socorro/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/peterbe/socorro/labels{/name}","releases_url":"https://api.github.com/repos/peterbe/socorro/releases{/id}","deployments_url":"https://api.github.com/repos/peterbe/socorro/deployments","created_at":"2011-12-05T23:07:30Z","updated_at":"2014-06-24T00:26:16Z","pushed_at":"2016-03-29T13:31:24Z","git_url":"git://github.com/peterbe/socorro.git","ssh_url":"git@github.com:peterbe/socorro.git","clone_url":"https://github.com/peterbe/socorro.git","svn_url":"https://github.com/peterbe/socorro","homepage":"https://wiki.mozilla.org/Socorro","size":163767,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":1,"default_branch":"master"}},"base":{"label":"mozilla:master","ref":"master","sha":"e95020bd93e9edf5ddbc5903074e6ba7b9f38246","user":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"repo":{"id":2263742,"name":"socorro","full_name":"mozilla/socorro","owner":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mozilla/socorro","description":"Socorro is a server to accept and process Breakpad crash reports. ","fork":false,"url":"https://api.github.com/repos/mozilla/socorro","forks_url":"https://api.github.com/repos/mozilla/socorro/forks","keys_url":"https://api.github.com/repos/mozilla/socorro/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mozilla/socorro/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mozilla/socorro/teams","hooks_url":"https://api.github.com/repos/mozilla/socorro/hooks","issue_events_url":"https://api.github.com/repos/mozilla/socorro/issues/events{/number}","events_url":"https://api.github.com/repos/mozilla/socorro/events","assignees_url":"https://api.github.com/repos/mozilla/socorro/assignees{/user}","branches_url":"https://api.github.com/repos/mozilla/socorro/branches{/branch}","tags_url":"https://api.github.com/repos/mozilla/socorro/tags","blobs_url":"https://api.github.com/repos/mozilla/socorro/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mozilla/socorro/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mozilla/socorro/git/refs{/sha}","trees_url":"https://api.github.com/repos/mozilla/socorro/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mozilla/socorro/statuses/{sha}","languages_url":"https://api.github.com/repos/mozilla/socorro/languages","stargazers_url":"https://api.github.com/repos/mozilla/socorro/stargazers","contributors_url":"https://api.github.com/repos/mozilla/socorro/contributors","subscribers_url":"https://api.github.com/repos/mozilla/socorro/subscribers","subscription_url":"https://api.github.com/repos/mozilla/socorro/subscription","commits_url":"https://api.github.com/repos/mozilla/socorro/commits{/sha}","git_commits_url":"https://api.github.com/repos/mozilla/socorro/git/commits{/sha}","comments_url":"https://api.github.com/repos/mozilla/socorro/comments{/number}","issue_comment_url":"https://api.github.com/repos/mozilla/socorro/issues/comments{/number}","contents_url":"https://api.github.com/repos/mozilla/socorro/contents/{+path}","compare_url":"https://api.github.com/repos/mozilla/socorro/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mozilla/socorro/merges","archive_url":"https://api.github.com/repos/mozilla/socorro/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mozilla/socorro/downloads","issues_url":"https://api.github.com/repos/mozilla/socorro/issues{/number}","pulls_url":"https://api.github.com/repos/mozilla/socorro/pulls{/number}","milestones_url":"https://api.github.com/repos/mozilla/socorro/milestones{/number}","notifications_url":"https://api.github.com/repos/mozilla/socorro/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mozilla/socorro/labels{/name}","releases_url":"https://api.github.com/repos/mozilla/socorro/releases{/id}","deployments_url":"https://api.github.com/repos/mozilla/socorro/deployments","created_at":"2011-08-24T19:21:49Z","updated_at":"2016-03-24T20:07:45Z","pushed_at":"2016-03-29T13:31:25Z","git_url":"git://github.com/mozilla/socorro.git","ssh_url":"git@github.com:mozilla/socorro.git","clone_url":"https://github.com/mozilla/socorro.git","svn_url":"https://github.com/mozilla/socorro","homepage":"https://wiki.mozilla.org/Socorro","size":166174,"stargazers_count":269,"watchers_count":269,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":144,"mirror_url":null,"open_issues_count":6,"forks":144,"open_issues":6,"watchers":269,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/mozilla/socorro/pulls/3251"},"html":{"href":"https://github.com/mozilla/socorro/pull/3251"},"issue":{"href":"https://api.github.com/repos/mozilla/socorro/issues/3251"},"comments":{"href":"https://api.github.com/repos/mozilla/socorro/issues/3251/comments"},"review_comments":{"href":"https://api.github.com/repos/mozilla/socorro/pulls/3251/comments"},"review_comment":{"href":"https://api.github.com/repos/mozilla/socorro/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/mozilla/socorro/pulls/3251/commits"},"statuses":{"href":"https://api.github.com/repos/mozilla/socorro/statuses/8388dab64f56aecb90b034d1ae566b9a545da476"}}}}