{"action":"created","comment":{"url":"https://api.github.com/repos/conversationai/wikidetox/pulls/comments/185763797","pull_request_review_id":117217204,"id":185763797,"diff_hunk":"@@ -72,7 +75,21 @@ def run(known_args, pipeline_args):\n   with beam.Pipeline(options=pipeline_options) as p:\n     pcoll = (p | ReadFromText(known_args.input)\n                    | beam.ParDo(WriteDecompressedFile())\n-                   | beam.io.Write(bigquery.BigQuerySink(known_args.table, schema=known_args.schema, validate = True)))\n+                   | beam.Map(lambda x: ('{week}at{year}'.format(week=x['week'], year=x['year']), x))\n+                   | beam.GroupByKey()\n+                   | beam.ParDo(WriteToStorage()))\n+\n+class WriteToStorage(beam.DoFn):\n+  def process(self, element):\n+      (key, val) = element\n+      week, year = [int(x) for x in key.split('at')]\n+      path = known_args.output + 'date-{week}at{year}/revisions.json'.format(week=week, year=year)","path":"conversation_reconstruction_dataflow/ingest_revisions/dataflow_main.py","position":63,"original_position":47,"commit_id":"39e52e4e78d4011bc5b9c68e5152de75bd42577a","original_commit_id":"72fb048932404fe98af621813924c985e57118a0","user":{"login":"sorensenjs","id":8582676,"avatar_url":"https://avatars1.githubusercontent.com/u/8582676?v=4","gravatar_id":"","url":"https://api.github.com/users/sorensenjs","html_url":"https://github.com/sorensenjs","followers_url":"https://api.github.com/users/sorensenjs/followers","following_url":"https://api.github.com/users/sorensenjs/following{/other_user}","gists_url":"https://api.github.com/users/sorensenjs/gists{/gist_id}","starred_url":"https://api.github.com/users/sorensenjs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sorensenjs/subscriptions","organizations_url":"https://api.github.com/users/sorensenjs/orgs","repos_url":"https://api.github.com/users/sorensenjs/repos","events_url":"https://api.github.com/users/sorensenjs/events{/privacy}","received_events_url":"https://api.github.com/users/sorensenjs/received_events","type":"User","site_admin":false},"body":"No custom sink needed - you can Partition() by year and Partition() by month and then use the provided Sinks. Or you can just Partition() by month and year directly. Writing sinks in dataflow is very hard.","created_at":"2018-05-03T11:09:11Z","updated_at":"2018-05-03T11:09:11Z","html_url":"https://github.com/conversationai/wikidetox/pull/88#discussion_r185763797","pull_request_url":"https://api.github.com/repos/conversationai/wikidetox/pulls/88","author_association":"NONE","_links":{"self":{"href":"https://api.github.com/repos/conversationai/wikidetox/pulls/comments/185763797"},"html":{"href":"https://github.com/conversationai/wikidetox/pull/88#discussion_r185763797"},"pull_request":{"href":"https://api.github.com/repos/conversationai/wikidetox/pulls/88"}},"in_reply_to_id":170164727},"pull_request":{"url":"https://api.github.com/repos/conversationai/wikidetox/pulls/88","id":170853718,"html_url":"https://github.com/conversationai/wikidetox/pull/88","diff_url":"https://github.com/conversationai/wikidetox/pull/88.diff","patch_url":"https://github.com/conversationai/wikidetox/pull/88.patch","issue_url":"https://api.github.com/repos/conversationai/wikidetox/issues/88","number":88,"state":"open","locked":false,"title":"Ingest revisions and shard storage by week","user":{"login":"vegetable68","id":2016736,"avatar_url":"https://avatars2.githubusercontent.com/u/2016736?v=4","gravatar_id":"","url":"https://api.github.com/users/vegetable68","html_url":"https://github.com/vegetable68","followers_url":"https://api.github.com/users/vegetable68/followers","following_url":"https://api.github.com/users/vegetable68/following{/other_user}","gists_url":"https://api.github.com/users/vegetable68/gists{/gist_id}","starred_url":"https://api.github.com/users/vegetable68/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vegetable68/subscriptions","organizations_url":"https://api.github.com/users/vegetable68/orgs","repos_url":"https://api.github.com/users/vegetable68/repos","events_url":"https://api.github.com/users/vegetable68/events{/privacy}","received_events_url":"https://api.github.com/users/vegetable68/received_events","type":"User","site_admin":false},"body":"- This implements a new ingestion pipeline that ingested all the revisions and operates a groupby according to the week of the timestamp and the revision.\r\n- Write the revision to cloud storage using a ParDo.\r\n\r\nDataflow Job here: https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-02-22_08_49_43-5186810640516118154?project=wikidetox-viz\r\n\r\nProblem: It seems like GroupByKey is waiting until all the data has finished processing and thus taking a lot of memory space.","created_at":"2018-02-22T20:12:16Z","updated_at":"2018-05-03T11:09:11Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5bcdfe0b33ed319ed8702f113719b9e8248f7e3c","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/conversationai/wikidetox/pulls/88/commits","review_comments_url":"https://api.github.com/repos/conversationai/wikidetox/pulls/88/comments","review_comment_url":"https://api.github.com/repos/conversationai/wikidetox/pulls/comments{/number}","comments_url":"https://api.github.com/repos/conversationai/wikidetox/issues/88/comments","statuses_url":"https://api.github.com/repos/conversationai/wikidetox/statuses/39e52e4e78d4011bc5b9c68e5152de75bd42577a","head":{"label":"conversationai:dataflow_pipeline","ref":"dataflow_pipeline","sha":"39e52e4e78d4011bc5b9c68e5152de75bd42577a","user":{"login":"conversationai","id":23317692,"avatar_url":"https://avatars1.githubusercontent.com/u/23317692?v=4","gravatar_id":"","url":"https://api.github.com/users/conversationai","html_url":"https://github.com/conversationai","followers_url":"https://api.github.com/users/conversationai/followers","following_url":"https://api.github.com/users/conversationai/following{/other_user}","gists_url":"https://api.github.com/users/conversationai/gists{/gist_id}","starred_url":"https://api.github.com/users/conversationai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/conversationai/subscriptions","organizations_url":"https://api.github.com/users/conversationai/orgs","repos_url":"https://api.github.com/users/conversationai/repos","events_url":"https://api.github.com/users/conversationai/events{/privacy}","received_events_url":"https://api.github.com/users/conversationai/received_events","type":"Organization","site_admin":false},"repo":{"id":98671024,"name":"wikidetox","full_name":"conversationai/wikidetox","owner":{"login":"conversationai","id":23317692,"avatar_url":"https://avatars1.githubusercontent.com/u/23317692?v=4","gravatar_id":"","url":"https://api.github.com/users/conversationai","html_url":"https://github.com/conversationai","followers_url":"https://api.github.com/users/conversationai/followers","following_url":"https://api.github.com/users/conversationai/following{/other_user}","gists_url":"https://api.github.com/users/conversationai/gists{/gist_id}","starred_url":"https://api.github.com/users/conversationai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/conversationai/subscriptions","organizations_url":"https://api.github.com/users/conversationai/orgs","repos_url":"https://api.github.com/users/conversationai/repos","events_url":"https://api.github.com/users/conversationai/events{/privacy}","received_events_url":"https://api.github.com/users/conversationai/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/conversationai/wikidetox","description":"Experiments to help discussion on Wikipedia talk pages","fork":false,"url":"https://api.github.com/repos/conversationai/wikidetox","forks_url":"https://api.github.com/repos/conversationai/wikidetox/forks","keys_url":"https://api.github.com/repos/conversationai/wikidetox/keys{/key_id}","collaborators_url":"https://api.github.com/repos/conversationai/wikidetox/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/conversationai/wikidetox/teams","hooks_url":"https://api.github.com/repos/conversationai/wikidetox/hooks","issue_events_url":"https://api.github.com/repos/conversationai/wikidetox/issues/events{/number}","events_url":"https://api.github.com/repos/conversationai/wikidetox/events","assignees_url":"https://api.github.com/repos/conversationai/wikidetox/assignees{/user}","branches_url":"https://api.github.com/repos/conversationai/wikidetox/branches{/branch}","tags_url":"https://api.github.com/repos/conversationai/wikidetox/tags","blobs_url":"https://api.github.com/repos/conversationai/wikidetox/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/conversationai/wikidetox/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/conversationai/wikidetox/git/refs{/sha}","trees_url":"https://api.github.com/repos/conversationai/wikidetox/git/trees{/sha}","statuses_url":"https://api.github.com/repos/conversationai/wikidetox/statuses/{sha}","languages_url":"https://api.github.com/repos/conversationai/wikidetox/languages","stargazers_url":"https://api.github.com/repos/conversationai/wikidetox/stargazers","contributors_url":"https://api.github.com/repos/conversationai/wikidetox/contributors","subscribers_url":"https://api.github.com/repos/conversationai/wikidetox/subscribers","subscription_url":"https://api.github.com/repos/conversationai/wikidetox/subscription","commits_url":"https://api.github.com/repos/conversationai/wikidetox/commits{/sha}","git_commits_url":"https://api.github.com/repos/conversationai/wikidetox/git/commits{/sha}","comments_url":"https://api.github.com/repos/conversationai/wikidetox/comments{/number}","issue_comment_url":"https://api.github.com/repos/conversationai/wikidetox/issues/comments{/number}","contents_url":"https://api.github.com/repos/conversationai/wikidetox/contents/{+path}","compare_url":"https://api.github.com/repos/conversationai/wikidetox/compare/{base}...{head}","merges_url":"https://api.github.com/repos/conversationai/wikidetox/merges","archive_url":"https://api.github.com/repos/conversationai/wikidetox/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/conversationai/wikidetox/downloads","issues_url":"https://api.github.com/repos/conversationai/wikidetox/issues{/number}","pulls_url":"https://api.github.com/repos/conversationai/wikidetox/pulls{/number}","milestones_url":"https://api.github.com/repos/conversationai/wikidetox/milestones{/number}","notifications_url":"https://api.github.com/repos/conversationai/wikidetox/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/conversationai/wikidetox/labels{/name}","releases_url":"https://api.github.com/repos/conversationai/wikidetox/releases{/id}","deployments_url":"https://api.github.com/repos/conversationai/wikidetox/deployments","created_at":"2017-07-28T17:11:51Z","updated_at":"2018-04-23T05:27:19Z","pushed_at":"2018-05-03T00:13:45Z","git_url":"git://github.com/conversationai/wikidetox.git","ssh_url":"git@github.com:conversationai/wikidetox.git","clone_url":"https://github.com/conversationai/wikidetox.git","svn_url":"https://github.com/conversationai/wikidetox","homepage":"","size":185222,"stargazers_count":12,"watchers_count":12,"language":"OpenEdge ABL","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":5,"mirror_url":null,"archived":false,"open_issues_count":11,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":5,"open_issues":11,"watchers":12,"default_branch":"master"}},"base":{"label":"conversationai:master","ref":"master","sha":"ce84fc9d2e14cd022e114e3a8e14c57ec5e5666d","user":{"login":"conversationai","id":23317692,"avatar_url":"https://avatars1.githubusercontent.com/u/23317692?v=4","gravatar_id":"","url":"https://api.github.com/users/conversationai","html_url":"https://github.com/conversationai","followers_url":"https://api.github.com/users/conversationai/followers","following_url":"https://api.github.com/users/conversationai/following{/other_user}","gists_url":"https://api.github.com/users/conversationai/gists{/gist_id}","starred_url":"https://api.github.com/users/conversationai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/conversationai/subscriptions","organizations_url":"https://api.github.com/users/conversationai/orgs","repos_url":"https://api.github.com/users/conversationai/repos","events_url":"https://api.github.com/users/conversationai/events{/privacy}","received_events_url":"https://api.github.com/users/conversationai/received_events","type":"Organization","site_admin":false},"repo":{"id":98671024,"name":"wikidetox","full_name":"conversationai/wikidetox","owner":{"login":"conversationai","id":23317692,"avatar_url":"https://avatars1.githubusercontent.com/u/23317692?v=4","gravatar_id":"","url":"https://api.github.com/users/conversationai","html_url":"https://github.com/conversationai","followers_url":"https://api.github.com/users/conversationai/followers","following_url":"https://api.github.com/users/conversationai/following{/other_user}","gists_url":"https://api.github.com/users/conversationai/gists{/gist_id}","starred_url":"https://api.github.com/users/conversationai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/conversationai/subscriptions","organizations_url":"https://api.github.com/users/conversationai/orgs","repos_url":"https://api.github.com/users/conversationai/repos","events_url":"https://api.github.com/users/conversationai/events{/privacy}","received_events_url":"https://api.github.com/users/conversationai/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/conversationai/wikidetox","description":"Experiments to help discussion on Wikipedia talk pages","fork":false,"url":"https://api.github.com/repos/conversationai/wikidetox","forks_url":"https://api.github.com/repos/conversationai/wikidetox/forks","keys_url":"https://api.github.com/repos/conversationai/wikidetox/keys{/key_id}","collaborators_url":"https://api.github.com/repos/conversationai/wikidetox/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/conversationai/wikidetox/teams","hooks_url":"https://api.github.com/repos/conversationai/wikidetox/hooks","issue_events_url":"https://api.github.com/repos/conversationai/wikidetox/issues/events{/number}","events_url":"https://api.github.com/repos/conversationai/wikidetox/events","assignees_url":"https://api.github.com/repos/conversationai/wikidetox/assignees{/user}","branches_url":"https://api.github.com/repos/conversationai/wikidetox/branches{/branch}","tags_url":"https://api.github.com/repos/conversationai/wikidetox/tags","blobs_url":"https://api.github.com/repos/conversationai/wikidetox/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/conversationai/wikidetox/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/conversationai/wikidetox/git/refs{/sha}","trees_url":"https://api.github.com/repos/conversationai/wikidetox/git/trees{/sha}","statuses_url":"https://api.github.com/repos/conversationai/wikidetox/statuses/{sha}","languages_url":"https://api.github.com/repos/conversationai/wikidetox/languages","stargazers_url":"https://api.github.com/repos/conversationai/wikidetox/stargazers","contributors_url":"https://api.github.com/repos/conversationai/wikidetox/contributors","subscribers_url":"https://api.github.com/repos/conversationai/wikidetox/subscribers","subscription_url":"https://api.github.com/repos/conversationai/wikidetox/subscription","commits_url":"https://api.github.com/repos/conversationai/wikidetox/commits{/sha}","git_commits_url":"https://api.github.com/repos/conversationai/wikidetox/git/commits{/sha}","comments_url":"https://api.github.com/repos/conversationai/wikidetox/comments{/number}","issue_comment_url":"https://api.github.com/repos/conversationai/wikidetox/issues/comments{/number}","contents_url":"https://api.github.com/repos/conversationai/wikidetox/contents/{+path}","compare_url":"https://api.github.com/repos/conversationai/wikidetox/compare/{base}...{head}","merges_url":"https://api.github.com/repos/conversationai/wikidetox/merges","archive_url":"https://api.github.com/repos/conversationai/wikidetox/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/conversationai/wikidetox/downloads","issues_url":"https://api.github.com/repos/conversationai/wikidetox/issues{/number}","pulls_url":"https://api.github.com/repos/conversationai/wikidetox/pulls{/number}","milestones_url":"https://api.github.com/repos/conversationai/wikidetox/milestones{/number}","notifications_url":"https://api.github.com/repos/conversationai/wikidetox/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/conversationai/wikidetox/labels{/name}","releases_url":"https://api.github.com/repos/conversationai/wikidetox/releases{/id}","deployments_url":"https://api.github.com/repos/conversationai/wikidetox/deployments","created_at":"2017-07-28T17:11:51Z","updated_at":"2018-04-23T05:27:19Z","pushed_at":"2018-05-03T00:13:45Z","git_url":"git://github.com/conversationai/wikidetox.git","ssh_url":"git@github.com:conversationai/wikidetox.git","clone_url":"https://github.com/conversationai/wikidetox.git","svn_url":"https://github.com/conversationai/wikidetox","homepage":"","size":185222,"stargazers_count":12,"watchers_count":12,"language":"OpenEdge ABL","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":5,"mirror_url":null,"archived":false,"open_issues_count":11,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":5,"open_issues":11,"watchers":12,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/conversationai/wikidetox/pulls/88"},"html":{"href":"https://github.com/conversationai/wikidetox/pull/88"},"issue":{"href":"https://api.github.com/repos/conversationai/wikidetox/issues/88"},"comments":{"href":"https://api.github.com/repos/conversationai/wikidetox/issues/88/comments"},"review_comments":{"href":"https://api.github.com/repos/conversationai/wikidetox/pulls/88/comments"},"review_comment":{"href":"https://api.github.com/repos/conversationai/wikidetox/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/conversationai/wikidetox/pulls/88/commits"},"statuses":{"href":"https://api.github.com/repos/conversationai/wikidetox/statuses/39e52e4e78d4011bc5b9c68e5152de75bd42577a"}},"author_association":"COLLABORATOR"}}
{"actor":{"display_login":"sorensenjs"}}