{"action":"created","comment":{"url":"https://api.github.com/repos/ftlabs/overview/pulls/comments/223644338","pull_request_review_id":162818936,"id":223644338,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIyMzY0NDMzOA==","diff_hunk":"@@ -348,6 +348,227 @@ function getArticleRelations(days, facets, aspects)\n \t\t});\n }\n \n+function getArticlesAggregation(days, facets, aspects, minCorrelation=2)\n+{\n+\tconst date = time.getDatetimeRange('days', days, 0);\n+\taspects = checkMetaDataPresent(aspects);\n+\tconst params = {\n+\t\tqueryString: `lastPublishDateTime:> ${date.next}`,\n+\t\tmaxResults : 100,\n+\t\taspects : aspects,\n+\t\tfacets : {\n+\t\t\t\"names\": facets,\n+\t\t\t\"maxElements\":-1\n+\t\t}\n+\t};\n+\n+\treturn searchDeeper(params, 10)\n+\t\t.then(results =>  extractArticles(results) )\n+\t\t.then(articles => aggregateArticles(articles, facets, minCorrelation, days, params, filterByMetadataCsv=null) )\n+\t\t.then(aggregation => {\n+\t\t\t// look for all the distinct genre CSVs\n+\t\t\tlet allCsvs = Object.keys(aggregation.articlesByMetadataCsv);\n+\t\t\tlet allGenreCsvs = allCsvs.filter( csv => csv.startsWith('genre:') );\n+\t\t\t// re-establish the full list of articles\n+\t\t\tlet articles = Object.values(aggregation.articlesByUuid);\n+\t\t\t// construct an aggregation for each genre csv\n+\t\t\tlet aggregationsByGenre = {};\n+\t\t\tallGenreCsvs.forEach( csv => {\n+\t\t\t\taggregationsByGenre[csv] = aggregateArticles(articles, facets, minCorrelation, days, params, filterByMetadataCsv=csv);\n+\t\t\t});\n+\t\t\tlet articleCounts = {};\n+\t\t\tallGenreCsvs.forEach(csv => {\n+\t\t\t\tarticleCounts[csv] = aggregationsByGenre[csv].numArticles;\n+\t\t\t});\n+\n+\t\t\treturn {\n+\t\t\t\t'description' :[\n+\t\t\t\t\t'calculating aggregations for each genre CSV',\n+\t\t\t\t],\n+\t\t\t\tarticleCounts,\n+\t\t\t\taggregationsByGenre\n+\t\t\t};\n+\t\t})\n+\t\t.catch(err => {\n+\t\t\tthrow err;\n+\t\t});\n+}\n+\n+function addUuidToMetadataTerm(grouping, metadataKey, term, uuid){\n+\tif (!grouping.hasOwnProperty(metadataKey)) {\n+\t\tgrouping[metadataKey] = {};\n+\t}\n+\n+\tif (!grouping[metadataKey].hasOwnProperty(term.taxonomy)) {\n+\t\tgrouping[metadataKey][term.taxonomy] = {};\n+\t}\n+\n+\tif (!grouping[metadataKey][term.taxonomy].hasOwnProperty(term.name)) {\n+\t\tgrouping[metadataKey][term.taxonomy][term.name] = [];\n+\t}\n+\n+\tgrouping[metadataKey][term.taxonomy][term.name].push(uuid);\n+}\n+\n+function addFacetCorrelation( grouping, metadataKey1, term1, metadataKey2, term2 ){\n+\tif (!grouping.hasOwnProperty(metadataKey1)) {\n+\t\tgrouping[metadataKey1] = {};\n+\t}\n+\n+\tif (!grouping[metadataKey1].hasOwnProperty(term1.taxonomy)) {\n+\t\tgrouping[metadataKey1][term1.taxonomy] = {};\n+\t}\n+\n+\tif (!grouping[metadataKey1][term1.taxonomy].hasOwnProperty(term1.name)) {\n+\t\tgrouping[metadataKey1][term1.taxonomy][term1.name] = {};\n+\t}\n+\n+\tlet csvTerm2 = [metadataKey2, term2.taxonomy, term2.name].join(':');\n+\n+\tif (!grouping[metadataKey1][term1.taxonomy][term1.name].hasOwnProperty(csvTerm2)) {\n+\t\tgrouping[metadataKey1][term1.taxonomy][term1.name][csvTerm2] = 0;\n+\t}\n+\n+\tgrouping[metadataKey1][term1.taxonomy][term1.name][csvTerm2]++;\n+}\n+\n+function aggregateArticles(articles, facets, minCorrelation, days, searchParams, filterByMetadataCsv=null)\n+{\n+\tlet articlesByUuid = {}; // UUID -> article\n+\tlet articlesByFacetTaxonomyName = {}; // {metadataKey}{taxonomy}{name} -> [UUID]\n+\tlet facetCorrelations = {}; // {metadataKey}{taxonomy}{name} {metadataKey:taxonomy:name} -> count\n+\n+\tarticles.forEach(article => {\n+\t\tlet uuid = article.id;","path":"lib/fetchContent.js","position":null,"original_position":95,"commit_id":"367faab0ee7cb6356135c1d1f24ca1681ed29ee0","original_commit_id":"5d35279d4f50292e531a3c9397b9947fea1d07a4","user":{"login":"railsagainstignorance","id":250310,"node_id":"MDQ6VXNlcjI1MDMxMA==","avatar_url":"https://avatars1.githubusercontent.com/u/250310?v=4","gravatar_id":"","url":"https://api.github.com/users/railsagainstignorance","html_url":"https://github.com/railsagainstignorance","followers_url":"https://api.github.com/users/railsagainstignorance/followers","following_url":"https://api.github.com/users/railsagainstignorance/following{/other_user}","gists_url":"https://api.github.com/users/railsagainstignorance/gists{/gist_id}","starred_url":"https://api.github.com/users/railsagainstignorance/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/railsagainstignorance/subscriptions","organizations_url":"https://api.github.com/users/railsagainstignorance/orgs","repos_url":"https://api.github.com/users/railsagainstignorance/repos","events_url":"https://api.github.com/users/railsagainstignorance/events{/privacy}","received_events_url":"https://api.github.com/users/railsagainstignorance/received_events","type":"User","site_admin":false},"body":"yep. done\r\n","created_at":"2018-10-09T10:33:07Z","updated_at":"2018-10-09T10:33:07Z","html_url":"https://github.com/ftlabs/overview/pull/19#discussion_r223644338","pull_request_url":"https://api.github.com/repos/ftlabs/overview/pulls/19","author_association":"NONE","_links":{"self":{"href":"https://api.github.com/repos/ftlabs/overview/pulls/comments/223644338"},"html":{"href":"https://github.com/ftlabs/overview/pull/19#discussion_r223644338"},"pull_request":{"href":"https://api.github.com/repos/ftlabs/overview/pulls/19"}},"in_reply_to_id":223635172},"pull_request":{"url":"https://api.github.com/repos/ftlabs/overview/pulls/19","id":221376400,"node_id":"MDExOlB1bGxSZXF1ZXN0MjIxMzc2NDAw","html_url":"https://github.com/ftlabs/overview/pull/19","diff_url":"https://github.com/ftlabs/overview/pull/19.diff","patch_url":"https://github.com/ftlabs/overview/pull/19.patch","issue_url":"https://api.github.com/repos/ftlabs/overview/issues/19","number":19,"state":"open","locked":false,"title":"Metadata aggregations","user":{"login":"railsagainstignorance","id":250310,"node_id":"MDQ6VXNlcjI1MDMxMA==","avatar_url":"https://avatars1.githubusercontent.com/u/250310?v=4","gravatar_id":"","url":"https://api.github.com/users/railsagainstignorance","html_url":"https://github.com/railsagainstignorance","followers_url":"https://api.github.com/users/railsagainstignorance/followers","following_url":"https://api.github.com/users/railsagainstignorance/following{/other_user}","gists_url":"https://api.github.com/users/railsagainstignorance/gists{/gist_id}","starred_url":"https://api.github.com/users/railsagainstignorance/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/railsagainstignorance/subscriptions","organizations_url":"https://api.github.com/users/railsagainstignorance/orgs","repos_url":"https://api.github.com/users/railsagainstignorance/repos","events_url":"https://api.github.com/users/railsagainstignorance/events{/privacy}","received_events_url":"https://api.github.com/users/railsagainstignorance/received_events","type":"User","site_admin":false},"body":"","created_at":"2018-10-09T09:56:10Z","updated_at":"2018-10-09T10:33:07Z","closed_at":null,"merged_at":null,"merge_commit_sha":"2ecdb5fec50be08abf6f4369f71028743ae2749f","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/ftlabs/overview/pulls/19/commits","review_comments_url":"https://api.github.com/repos/ftlabs/overview/pulls/19/comments","review_comment_url":"https://api.github.com/repos/ftlabs/overview/pulls/comments{/number}","comments_url":"https://api.github.com/repos/ftlabs/overview/issues/19/comments","statuses_url":"https://api.github.com/repos/ftlabs/overview/statuses/367faab0ee7cb6356135c1d1f24ca1681ed29ee0","head":{"label":"ftlabs:metadata","ref":"metadata","sha":"367faab0ee7cb6356135c1d1f24ca1681ed29ee0","user":{"login":"ftlabs","id":1432696,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE0MzI2OTY=","avatar_url":"https://avatars3.githubusercontent.com/u/1432696?v=4","gravatar_id":"","url":"https://api.github.com/users/ftlabs","html_url":"https://github.com/ftlabs","followers_url":"https://api.github.com/users/ftlabs/followers","following_url":"https://api.github.com/users/ftlabs/following{/other_user}","gists_url":"https://api.github.com/users/ftlabs/gists{/gist_id}","starred_url":"https://api.github.com/users/ftlabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ftlabs/subscriptions","organizations_url":"https://api.github.com/users/ftlabs/orgs","repos_url":"https://api.github.com/users/ftlabs/repos","events_url":"https://api.github.com/users/ftlabs/events{/privacy}","received_events_url":"https://api.github.com/users/ftlabs/received_events","type":"Organization","site_admin":false},"repo":{"id":143861158,"node_id":"MDEwOlJlcG9zaXRvcnkxNDM4NjExNTg=","name":"overview","full_name":"ftlabs/overview","private":false,"owner":{"login":"ftlabs","id":1432696,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE0MzI2OTY=","avatar_url":"https://avatars3.githubusercontent.com/u/1432696?v=4","gravatar_id":"","url":"https://api.github.com/users/ftlabs","html_url":"https://github.com/ftlabs","followers_url":"https://api.github.com/users/ftlabs/followers","following_url":"https://api.github.com/users/ftlabs/following{/other_user}","gists_url":"https://api.github.com/users/ftlabs/gists{/gist_id}","starred_url":"https://api.github.com/users/ftlabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ftlabs/subscriptions","organizations_url":"https://api.github.com/users/ftlabs/orgs","repos_url":"https://api.github.com/users/ftlabs/repos","events_url":"https://api.github.com/users/ftlabs/events{/privacy}","received_events_url":"https://api.github.com/users/ftlabs/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/ftlabs/overview","description":"Exploration(s) of summarising and displaying news","fork":false,"url":"https://api.github.com/repos/ftlabs/overview","forks_url":"https://api.github.com/repos/ftlabs/overview/forks","keys_url":"https://api.github.com/repos/ftlabs/overview/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ftlabs/overview/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ftlabs/overview/teams","hooks_url":"https://api.github.com/repos/ftlabs/overview/hooks","issue_events_url":"https://api.github.com/repos/ftlabs/overview/issues/events{/number}","events_url":"https://api.github.com/repos/ftlabs/overview/events","assignees_url":"https://api.github.com/repos/ftlabs/overview/assignees{/user}","branches_url":"https://api.github.com/repos/ftlabs/overview/branches{/branch}","tags_url":"https://api.github.com/repos/ftlabs/overview/tags","blobs_url":"https://api.github.com/repos/ftlabs/overview/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ftlabs/overview/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ftlabs/overview/git/refs{/sha}","trees_url":"https://api.github.com/repos/ftlabs/overview/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ftlabs/overview/statuses/{sha}","languages_url":"https://api.github.com/repos/ftlabs/overview/languages","stargazers_url":"https://api.github.com/repos/ftlabs/overview/stargazers","contributors_url":"https://api.github.com/repos/ftlabs/overview/contributors","subscribers_url":"https://api.github.com/repos/ftlabs/overview/subscribers","subscription_url":"https://api.github.com/repos/ftlabs/overview/subscription","commits_url":"https://api.github.com/repos/ftlabs/overview/commits{/sha}","git_commits_url":"https://api.github.com/repos/ftlabs/overview/git/commits{/sha}","comments_url":"https://api.github.com/repos/ftlabs/overview/comments{/number}","issue_comment_url":"https://api.github.com/repos/ftlabs/overview/issues/comments{/number}","contents_url":"https://api.github.com/repos/ftlabs/overview/contents/{+path}","compare_url":"https://api.github.com/repos/ftlabs/overview/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ftlabs/overview/merges","archive_url":"https://api.github.com/repos/ftlabs/overview/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ftlabs/overview/downloads","issues_url":"https://api.github.com/repos/ftlabs/overview/issues{/number}","pulls_url":"https://api.github.com/repos/ftlabs/overview/pulls{/number}","milestones_url":"https://api.github.com/repos/ftlabs/overview/milestones{/number}","notifications_url":"https://api.github.com/repos/ftlabs/overview/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ftlabs/overview/labels{/name}","releases_url":"https://api.github.com/repos/ftlabs/overview/releases{/id}","deployments_url":"https://api.github.com/repos/ftlabs/overview/deployments","created_at":"2018-08-07T11:02:52Z","updated_at":"2018-10-09T09:13:46Z","pushed_at":"2018-10-09T10:32:21Z","git_url":"git://github.com/ftlabs/overview.git","ssh_url":"git@github.com:ftlabs/overview.git","clone_url":"https://github.com/ftlabs/overview.git","svn_url":"https://github.com/ftlabs/overview","homepage":null,"size":599,"stargazers_count":1,"watchers_count":1,"language":"JavaScript","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":2,"license":null,"forks":0,"open_issues":2,"watchers":1,"default_branch":"master"}},"base":{"label":"ftlabs:master","ref":"master","sha":"e81d57d7177ad0f75605f916acc5813c582f6bf8","user":{"login":"ftlabs","id":1432696,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE0MzI2OTY=","avatar_url":"https://avatars3.githubusercontent.com/u/1432696?v=4","gravatar_id":"","url":"https://api.github.com/users/ftlabs","html_url":"https://github.com/ftlabs","followers_url":"https://api.github.com/users/ftlabs/followers","following_url":"https://api.github.com/users/ftlabs/following{/other_user}","gists_url":"https://api.github.com/users/ftlabs/gists{/gist_id}","starred_url":"https://api.github.com/users/ftlabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ftlabs/subscriptions","organizations_url":"https://api.github.com/users/ftlabs/orgs","repos_url":"https://api.github.com/users/ftlabs/repos","events_url":"https://api.github.com/users/ftlabs/events{/privacy}","received_events_url":"https://api.github.com/users/ftlabs/received_events","type":"Organization","site_admin":false},"repo":{"id":143861158,"node_id":"MDEwOlJlcG9zaXRvcnkxNDM4NjExNTg=","name":"overview","full_name":"ftlabs/overview","private":false,"owner":{"login":"ftlabs","id":1432696,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE0MzI2OTY=","avatar_url":"https://avatars3.githubusercontent.com/u/1432696?v=4","gravatar_id":"","url":"https://api.github.com/users/ftlabs","html_url":"https://github.com/ftlabs","followers_url":"https://api.github.com/users/ftlabs/followers","following_url":"https://api.github.com/users/ftlabs/following{/other_user}","gists_url":"https://api.github.com/users/ftlabs/gists{/gist_id}","starred_url":"https://api.github.com/users/ftlabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ftlabs/subscriptions","organizations_url":"https://api.github.com/users/ftlabs/orgs","repos_url":"https://api.github.com/users/ftlabs/repos","events_url":"https://api.github.com/users/ftlabs/events{/privacy}","received_events_url":"https://api.github.com/users/ftlabs/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/ftlabs/overview","description":"Exploration(s) of summarising and displaying news","fork":false,"url":"https://api.github.com/repos/ftlabs/overview","forks_url":"https://api.github.com/repos/ftlabs/overview/forks","keys_url":"https://api.github.com/repos/ftlabs/overview/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ftlabs/overview/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ftlabs/overview/teams","hooks_url":"https://api.github.com/repos/ftlabs/overview/hooks","issue_events_url":"https://api.github.com/repos/ftlabs/overview/issues/events{/number}","events_url":"https://api.github.com/repos/ftlabs/overview/events","assignees_url":"https://api.github.com/repos/ftlabs/overview/assignees{/user}","branches_url":"https://api.github.com/repos/ftlabs/overview/branches{/branch}","tags_url":"https://api.github.com/repos/ftlabs/overview/tags","blobs_url":"https://api.github.com/repos/ftlabs/overview/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ftlabs/overview/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ftlabs/overview/git/refs{/sha}","trees_url":"https://api.github.com/repos/ftlabs/overview/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ftlabs/overview/statuses/{sha}","languages_url":"https://api.github.com/repos/ftlabs/overview/languages","stargazers_url":"https://api.github.com/repos/ftlabs/overview/stargazers","contributors_url":"https://api.github.com/repos/ftlabs/overview/contributors","subscribers_url":"https://api.github.com/repos/ftlabs/overview/subscribers","subscription_url":"https://api.github.com/repos/ftlabs/overview/subscription","commits_url":"https://api.github.com/repos/ftlabs/overview/commits{/sha}","git_commits_url":"https://api.github.com/repos/ftlabs/overview/git/commits{/sha}","comments_url":"https://api.github.com/repos/ftlabs/overview/comments{/number}","issue_comment_url":"https://api.github.com/repos/ftlabs/overview/issues/comments{/number}","contents_url":"https://api.github.com/repos/ftlabs/overview/contents/{+path}","compare_url":"https://api.github.com/repos/ftlabs/overview/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ftlabs/overview/merges","archive_url":"https://api.github.com/repos/ftlabs/overview/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ftlabs/overview/downloads","issues_url":"https://api.github.com/repos/ftlabs/overview/issues{/number}","pulls_url":"https://api.github.com/repos/ftlabs/overview/pulls{/number}","milestones_url":"https://api.github.com/repos/ftlabs/overview/milestones{/number}","notifications_url":"https://api.github.com/repos/ftlabs/overview/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ftlabs/overview/labels{/name}","releases_url":"https://api.github.com/repos/ftlabs/overview/releases{/id}","deployments_url":"https://api.github.com/repos/ftlabs/overview/deployments","created_at":"2018-08-07T11:02:52Z","updated_at":"2018-10-09T09:13:46Z","pushed_at":"2018-10-09T10:32:21Z","git_url":"git://github.com/ftlabs/overview.git","ssh_url":"git@github.com:ftlabs/overview.git","clone_url":"https://github.com/ftlabs/overview.git","svn_url":"https://github.com/ftlabs/overview","homepage":null,"size":599,"stargazers_count":1,"watchers_count":1,"language":"JavaScript","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":2,"license":null,"forks":0,"open_issues":2,"watchers":1,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/ftlabs/overview/pulls/19"},"html":{"href":"https://github.com/ftlabs/overview/pull/19"},"issue":{"href":"https://api.github.com/repos/ftlabs/overview/issues/19"},"comments":{"href":"https://api.github.com/repos/ftlabs/overview/issues/19/comments"},"review_comments":{"href":"https://api.github.com/repos/ftlabs/overview/pulls/19/comments"},"review_comment":{"href":"https://api.github.com/repos/ftlabs/overview/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/ftlabs/overview/pulls/19/commits"},"statuses":{"href":"https://api.github.com/repos/ftlabs/overview/statuses/367faab0ee7cb6356135c1d1f24ca1681ed29ee0"}},"author_association":"NONE"}}
{"actor":{"display_login":"railsagainstignorance"}}