{"action":"created","comment":{"url":"https://api.github.com/repos/mozilla/mozilla-reports/pulls/comments/93356567","pull_request_review_id":13882057,"id":93356567,"diff_hunk":"@@ -433,182 +401,188 @@ def get_newest_per_client(df):\n     )\n     return selectable_by_client.filter(selectable_by_client['row_number'] == 1)\n \n-DO_UPLOAD = True\n-\n-world_start = '20151101'\n-today = _datetime.utcnow()\n-todays = fmt(today)\n-wsd = _datetime.strptime(world_start, \"%Y%m%d\")\n-week_start_date = wsd\n-week_end_date = week_start_date + timedelta(6)\n-week_start = fmt(week_start_date)\n-week_end = fmt(week_end_date)\n \n-client = boto3.client('s3', 'us-west-2')\n-transfer = S3Transfer(client)\n-bucket = \"net-mozaws-prod-us-west-2-pipeline-analysis\"\n-\n-MAX_SUBSESSION_LENGTH = 60 * 60 * 48 # 48 hours in seconds.\n+def upload_to_s3(records, churn_outfile):\n+    client = boto3.client('s3', 'us-west-2')\n+    transfer = S3Transfer(client)\n+    churn_bucket = \"net-mozaws-prod-us-west-2-pipeline-analysis\"\n+    churn_filepath = \"amiyaguchi/churn\"\n+\n+    print \"{}: Writing output to {}\".format(_datetime.utcnow(), churn_outfile)\n+\n+    # Write the file out as gzipped csv\n+    with gzip.open(churn_outfile, 'wb') as fout:\n+        fout.write(\",\".join(record_columns) + \"\\n\")\n+        print \"{}: Wrote header to {}\".format(_datetime.utcnow(), churn_outfile)\n+        for r in records:\n+            try:\n+                fout.write(csv(r))\n+                fout.write(\"\\n\")\n+            except UnicodeEncodeError as e:\n+                print(\"{}: Error writing line: {} // {}\"\n+                      .format(_datetime.utcnow(), e, r))\n+        print \"{}: finished writing lines\".format(_datetime.utcnow())\n+\n+    # Now upload it to S3:","path":"etl/churn.kp/knowledge.md","position":207,"original_position":207,"commit_id":"7fe5ff3d508686c75600efa4630962bc81b29034","original_commit_id":"7fe5ff3d508686c75600efa4630962bc81b29034","user":{"login":"acmiyaguchi","id":3304040,"avatar_url":"https://avatars.githubusercontent.com/u/3304040?v=3","gravatar_id":"","url":"https://api.github.com/users/acmiyaguchi","html_url":"https://github.com/acmiyaguchi","followers_url":"https://api.github.com/users/acmiyaguchi/followers","following_url":"https://api.github.com/users/acmiyaguchi/following{/other_user}","gists_url":"https://api.github.com/users/acmiyaguchi/gists{/gist_id}","starred_url":"https://api.github.com/users/acmiyaguchi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/acmiyaguchi/subscriptions","organizations_url":"https://api.github.com/users/acmiyaguchi/orgs","repos_url":"https://api.github.com/users/acmiyaguchi/repos","events_url":"https://api.github.com/users/acmiyaguchi/events{/privacy}","received_events_url":"https://api.github.com/users/acmiyaguchi/received_events","type":"User","site_admin":false},"body":"Uploads to s3 can cause the job to fail if the object already exists. I should check for existence in both instances of uploading.","created_at":"2016-12-21T00:53:50Z","updated_at":"2016-12-21T00:53:50Z","html_url":"https://github.com/mozilla/mozilla-reports/pull/6#discussion_r93356567","pull_request_url":"https://api.github.com/repos/mozilla/mozilla-reports/pulls/6","_links":{"self":{"href":"https://api.github.com/repos/mozilla/mozilla-reports/pulls/comments/93356567"},"html":{"href":"https://github.com/mozilla/mozilla-reports/pull/6#discussion_r93356567"},"pull_request":{"href":"https://api.github.com/repos/mozilla/mozilla-reports/pulls/6"}}},"pull_request":{"url":"https://api.github.com/repos/mozilla/mozilla-reports/pulls/6","id":98684403,"html_url":"https://github.com/mozilla/mozilla-reports/pull/6","diff_url":"https://github.com/mozilla/mozilla-reports/pull/6.diff","patch_url":"https://github.com/mozilla/mozilla-reports/pull/6.patch","issue_url":"https://api.github.com/repos/mozilla/mozilla-reports/issues/6","number":6,"state":"open","locked":false,"title":"Bug 1321414 -Create v2 desktop churn/retention dataset in telemetry-parquet","user":{"login":"acmiyaguchi","id":3304040,"avatar_url":"https://avatars.githubusercontent.com/u/3304040?v=3","gravatar_id":"","url":"https://api.github.com/users/acmiyaguchi","html_url":"https://github.com/acmiyaguchi","followers_url":"https://api.github.com/users/acmiyaguchi/followers","following_url":"https://api.github.com/users/acmiyaguchi/following{/other_user}","gists_url":"https://api.github.com/users/acmiyaguchi/gists{/gist_id}","starred_url":"https://api.github.com/users/acmiyaguchi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/acmiyaguchi/subscriptions","organizations_url":"https://api.github.com/users/acmiyaguchi/orgs","repos_url":"https://api.github.com/users/acmiyaguchi/repos","events_url":"https://api.github.com/users/acmiyaguchi/events{/privacy}","received_events_url":"https://api.github.com/users/acmiyaguchi/received_events","type":"User","site_admin":false},"body":"This continues the discussion from [this pull request](https://github.com/mozilla-services/data-pipeline/pull/236).\r\n\r\nIn short, this prepares the churn dataset for being scheduled on airflow and to be available on redash. The last thing to do is to change the s3 paths.\r\n\r\nI've compared the results in the base version of this notebook (with the submission_date_s3 patch) with the results from this notebook. There are no significant changes in results from the original script. All changed rows are the result of small rounding errors. \r\n\r\n```\r\nDiff of original and patched in daterange 20161113-20161119\r\n+ csvdiff --style=summary channel,geo,is_funnelcake,acquisition_period,start_version,start_version,sync_usage,current_version,current_week,is_active churn-20161113-20161119.by_activity_original.csv churn-20161113-20161119.by_activity.csv\r\n1 rows removed (0.0%)\r\n1 rows added (0.0%)\r\n930 rows changed (0.0%)\r\n\r\nDiff of original and patched in daterange 20161120-20161126\r\n+ csvdiff --style=summary channel,geo,is_funnelcake,acquisition_period,start_version,start_version,sync_usage,current_version,current_week,is_active churn-20161120-20161126.by_activity_original.csv churn-20161120-20161126.by_activity.csv\r\n1 rows removed (0.0%)\r\n1 rows added (0.0%)\r\n949 rows changed (0.0%)\r\n\r\nDiff of original and patched in daterange 20161127-20161203\r\n+ csvdiff --style=summary channel,geo,is_funnelcake,acquisition_period,start_version,start_version,sync_usage,current_version,current_week,is_active churn-20161127-20161203.by_activity_original.csv churn-20161127-20161203.by_activity.csv\r\n1 rows removed (0.0%)\r\n1 rows added (0.0%)\r\n1133 rows changed (0.0%)\r\n```\r\n\r\nOn a side note, itts interesting to note that the change to using submission_date_s3 causes roughly a 1% difference in results.\r\n\r\n```\r\ncsvdiff --style=summary channel,geo,is_funnelcake,acquisition_period,start_version,start_version,sync_usage,current_version,current_week,is_active churn-20161009-20161015.by_activity_mreid.csv\r\nchurn-20161009-20161015.by_activity_testing.csv\r\n\r\n21395 rows removed (1.0%)\r\n25274 rows added (1.2%)\r\n26267 rows changed (1.3%)\r\n```\r\n\r\nr? @Dexterp37 @mreid-moz","created_at":"2016-12-20T04:20:24Z","updated_at":"2016-12-21T00:53:50Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7b2a0d08963595fc85bcf7bfc75cc9e2af60d352","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/mozilla/mozilla-reports/pulls/6/commits","review_comments_url":"https://api.github.com/repos/mozilla/mozilla-reports/pulls/6/comments","review_comment_url":"https://api.github.com/repos/mozilla/mozilla-reports/pulls/comments{/number}","comments_url":"https://api.github.com/repos/mozilla/mozilla-reports/issues/6/comments","statuses_url":"https://api.github.com/repos/mozilla/mozilla-reports/statuses/7fe5ff3d508686c75600efa4630962bc81b29034","head":{"label":"acmiyaguchi:etl/churn.kp","ref":"etl/churn.kp","sha":"7fe5ff3d508686c75600efa4630962bc81b29034","user":{"login":"acmiyaguchi","id":3304040,"avatar_url":"https://avatars.githubusercontent.com/u/3304040?v=3","gravatar_id":"","url":"https://api.github.com/users/acmiyaguchi","html_url":"https://github.com/acmiyaguchi","followers_url":"https://api.github.com/users/acmiyaguchi/followers","following_url":"https://api.github.com/users/acmiyaguchi/following{/other_user}","gists_url":"https://api.github.com/users/acmiyaguchi/gists{/gist_id}","starred_url":"https://api.github.com/users/acmiyaguchi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/acmiyaguchi/subscriptions","organizations_url":"https://api.github.com/users/acmiyaguchi/orgs","repos_url":"https://api.github.com/users/acmiyaguchi/repos","events_url":"https://api.github.com/users/acmiyaguchi/events{/privacy}","received_events_url":"https://api.github.com/users/acmiyaguchi/received_events","type":"User","site_admin":false},"repo":{"id":75128877,"name":"mozilla-reports","full_name":"acmiyaguchi/mozilla-reports","owner":{"login":"acmiyaguchi","id":3304040,"avatar_url":"https://avatars.githubusercontent.com/u/3304040?v=3","gravatar_id":"","url":"https://api.github.com/users/acmiyaguchi","html_url":"https://github.com/acmiyaguchi","followers_url":"https://api.github.com/users/acmiyaguchi/followers","following_url":"https://api.github.com/users/acmiyaguchi/following{/other_user}","gists_url":"https://api.github.com/users/acmiyaguchi/gists{/gist_id}","starred_url":"https://api.github.com/users/acmiyaguchi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/acmiyaguchi/subscriptions","organizations_url":"https://api.github.com/users/acmiyaguchi/orgs","repos_url":"https://api.github.com/users/acmiyaguchi/repos","events_url":"https://api.github.com/users/acmiyaguchi/events{/privacy}","received_events_url":"https://api.github.com/users/acmiyaguchi/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/acmiyaguchi/mozilla-reports","description":"Repository for public analyses.","fork":true,"url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports","forks_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/forks","keys_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/keys{/key_id}","collaborators_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/teams","hooks_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/hooks","issue_events_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/issues/events{/number}","events_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/events","assignees_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/assignees{/user}","branches_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/branches{/branch}","tags_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/tags","blobs_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/git/refs{/sha}","trees_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/git/trees{/sha}","statuses_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/statuses/{sha}","languages_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/languages","stargazers_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/stargazers","contributors_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/contributors","subscribers_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/subscribers","subscription_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/subscription","commits_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/commits{/sha}","git_commits_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/git/commits{/sha}","comments_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/comments{/number}","issue_comment_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/issues/comments{/number}","contents_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/contents/{+path}","compare_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/compare/{base}...{head}","merges_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/merges","archive_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/downloads","issues_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/issues{/number}","pulls_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/pulls{/number}","milestones_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/milestones{/number}","notifications_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/labels{/name}","releases_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/releases{/id}","deployments_url":"https://api.github.com/repos/acmiyaguchi/mozilla-reports/deployments","created_at":"2016-11-29T22:41:01Z","updated_at":"2016-11-29T22:41:03Z","pushed_at":"2016-12-20T19:44:40Z","git_url":"git://github.com/acmiyaguchi/mozilla-reports.git","ssh_url":"git@github.com:acmiyaguchi/mozilla-reports.git","clone_url":"https://github.com/acmiyaguchi/mozilla-reports.git","svn_url":"https://github.com/acmiyaguchi/mozilla-reports","homepage":"","size":137,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"mozilla:master","ref":"master","sha":"e9ad6637b501e80027c48f7a8a1bce5f4ca14063","user":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"repo":{"id":74477238,"name":"mozilla-reports","full_name":"mozilla/mozilla-reports","owner":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mozilla/mozilla-reports","description":"Repository for public analyses.","fork":false,"url":"https://api.github.com/repos/mozilla/mozilla-reports","forks_url":"https://api.github.com/repos/mozilla/mozilla-reports/forks","keys_url":"https://api.github.com/repos/mozilla/mozilla-reports/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mozilla/mozilla-reports/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mozilla/mozilla-reports/teams","hooks_url":"https://api.github.com/repos/mozilla/mozilla-reports/hooks","issue_events_url":"https://api.github.com/repos/mozilla/mozilla-reports/issues/events{/number}","events_url":"https://api.github.com/repos/mozilla/mozilla-reports/events","assignees_url":"https://api.github.com/repos/mozilla/mozilla-reports/assignees{/user}","branches_url":"https://api.github.com/repos/mozilla/mozilla-reports/branches{/branch}","tags_url":"https://api.github.com/repos/mozilla/mozilla-reports/tags","blobs_url":"https://api.github.com/repos/mozilla/mozilla-reports/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mozilla/mozilla-reports/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mozilla/mozilla-reports/git/refs{/sha}","trees_url":"https://api.github.com/repos/mozilla/mozilla-reports/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mozilla/mozilla-reports/statuses/{sha}","languages_url":"https://api.github.com/repos/mozilla/mozilla-reports/languages","stargazers_url":"https://api.github.com/repos/mozilla/mozilla-reports/stargazers","contributors_url":"https://api.github.com/repos/mozilla/mozilla-reports/contributors","subscribers_url":"https://api.github.com/repos/mozilla/mozilla-reports/subscribers","subscription_url":"https://api.github.com/repos/mozilla/mozilla-reports/subscription","commits_url":"https://api.github.com/repos/mozilla/mozilla-reports/commits{/sha}","git_commits_url":"https://api.github.com/repos/mozilla/mozilla-reports/git/commits{/sha}","comments_url":"https://api.github.com/repos/mozilla/mozilla-reports/comments{/number}","issue_comment_url":"https://api.github.com/repos/mozilla/mozilla-reports/issues/comments{/number}","contents_url":"https://api.github.com/repos/mozilla/mozilla-reports/contents/{+path}","compare_url":"https://api.github.com/repos/mozilla/mozilla-reports/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mozilla/mozilla-reports/merges","archive_url":"https://api.github.com/repos/mozilla/mozilla-reports/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mozilla/mozilla-reports/downloads","issues_url":"https://api.github.com/repos/mozilla/mozilla-reports/issues{/number}","pulls_url":"https://api.github.com/repos/mozilla/mozilla-reports/pulls{/number}","milestones_url":"https://api.github.com/repos/mozilla/mozilla-reports/milestones{/number}","notifications_url":"https://api.github.com/repos/mozilla/mozilla-reports/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mozilla/mozilla-reports/labels{/name}","releases_url":"https://api.github.com/repos/mozilla/mozilla-reports/releases{/id}","deployments_url":"https://api.github.com/repos/mozilla/mozilla-reports/deployments","created_at":"2016-11-22T13:53:59Z","updated_at":"2016-12-17T15:27:51Z","pushed_at":"2016-12-20T19:44:40Z","git_url":"git://github.com/mozilla/mozilla-reports.git","ssh_url":"git@github.com:mozilla/mozilla-reports.git","clone_url":"https://github.com/mozilla/mozilla-reports.git","svn_url":"https://github.com/mozilla/mozilla-reports","homepage":"","size":121,"stargazers_count":2,"watchers_count":2,"language":"Jupyter Notebook","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":2,"forks":1,"open_issues":2,"watchers":2,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/mozilla/mozilla-reports/pulls/6"},"html":{"href":"https://github.com/mozilla/mozilla-reports/pull/6"},"issue":{"href":"https://api.github.com/repos/mozilla/mozilla-reports/issues/6"},"comments":{"href":"https://api.github.com/repos/mozilla/mozilla-reports/issues/6/comments"},"review_comments":{"href":"https://api.github.com/repos/mozilla/mozilla-reports/pulls/6/comments"},"review_comment":{"href":"https://api.github.com/repos/mozilla/mozilla-reports/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/mozilla/mozilla-reports/pulls/6/commits"},"statuses":{"href":"https://api.github.com/repos/mozilla/mozilla-reports/statuses/7fe5ff3d508686c75600efa4630962bc81b29034"}}}}
{"actor":{"display_login":"acmiyaguchi"}}