{"action":"created","comment":{"url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/pulls/comments/182407800","pull_request_review_id":113195161,"id":182407800,"diff_hunk":"@@ -1,34 +1,110 @@\n #!/usr/bin/env python\n-# Requirements: python-dateutil, numpy, requests\n \n-from __future__ import print_function\n-from collections import defaultdict, namedtuple\n-import csv\n import dateutil.parser\n-import json\n-import numpy\n-import re\n import requests\n-import subprocess\n \n-from wpt_common import CUTOFF, QUARTER_START, fetch_all_prs, wpt_git\n+from csv_database import RunLatencyDB\n+from wpt_common import fetch_all_prs, get_pr_latencies\n \n-RUNS_URL = 'https://wpt.fyi/api/runs?max-count=100'\n-CSV_FILE = 'wpt-dashboard-latency.csv'\n+# 1000 because of https://github.com/w3c/wptdashboard/issues/524\n+RUNS_URL = 'https://wpt.fyi/api/runs?max-count=1000'\n+CSV_PATH_TEMPLATE = 'wpt-dashboard-{}-latencies.csv'\n \n+# If the runs expand to non-desktop platforms, those should be measured\n+# separately, so assert that only desktop configurations exist.\n+KNOWN_RUN_CONFIGS = set([\n+    ('chrome', 'debian'),\n+    ('chrome', 'linux'),\n+    ('chrome', 'linux\\n'),\n+    ('edge', 'windows'),\n+    ('firefox', 'debian'),\n+    ('firefox', 'linux'),\n+    ('safari', 'macos'),\n+    ('safari', 'macOS'),\n+])\n \n-def get_latencies(prs, runs):\n-    \"\"\"For each PR, find the earliest run for each browser that included that PR,\n-    and calucate the latencies between the PR and the runs.\"\"\"\n-    latencies = {}\n-    return latencies\n+\n+def run_sha(run):\n+    return run['revision']\n+\n+\n+def run_date(run):\n+    return dateutil.parser.parse(run['created_at'])\n+\n+\n+def filter_runs(runs, sort_key=None, sort_reverse=False, filter_key=None):\n+    \"\"\"Returns a list of runs filtered to the first (by sort_key) run\n+    considered a duplicate by filter_key.\"\"\"\n+\n+    filtered_runs = {}\n+    for run in sorted(runs, key=sort_key, reverse=sort_reverse):\n+        key = filter_key(run)\n+        if not filtered_runs.has_key(key):\n+            filtered_runs[key] = run\n+    return filtered_runs.values()\n+\n+\n+def write_latencies(prs, runs, name):\n+    \"\"\"For each PR, find the earliest run that included that PR, and calucate\n+    the latencies between the PR and the runs. Write the results to a CSV\n+    file.\"\"\"\n+\n+    latencies = get_pr_latencies(\n+        prs, events=runs, event_sha=run_sha, event_date=run_date)\n+    csv_path = CSV_PATH_TEMPLATE.format(name)\n+    db = RunLatencyDB(csv_path)\n+    for entry in latencies:\n+        run = entry['event']\n+        if run is None:\n+            continue\n+        db.add({\n+            'PR': entry['pr']['PR'],\n+            'run_sha': run['revision'],\n+            'run_time': run['created_at'],\n+            'latency': entry['latency'],\n+        })\n+    db.write()\n \n \n def main():\n-    prs = fetch_all_prs()\n-    runs = requests.get(RUNS_URL).json()\n-    latencies = get_latencies(prs, runs)\n-    print(latencies)\n+    prs = fetch_all_prs().values()\n+    runs_response = requests.get(RUNS_URL)\n+    runs_response.raise_for_status()\n+    runs = runs_response.json()\n+","path":"wpt-dashboard-stats.py","position":94,"original_position":94,"commit_id":"126faef342e893df96e15d3a58fdeebe2658a360","original_commit_id":"a08dabc4809961ed56ea9bf109294404204f4583","user":{"login":"foolip","id":498917,"avatar_url":"https://avatars1.githubusercontent.com/u/498917?v=4","gravatar_id":"","url":"https://api.github.com/users/foolip","html_url":"https://github.com/foolip","followers_url":"https://api.github.com/users/foolip/followers","following_url":"https://api.github.com/users/foolip/following{/other_user}","gists_url":"https://api.github.com/users/foolip/gists{/gist_id}","starred_url":"https://api.github.com/users/foolip/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/foolip/subscriptions","organizations_url":"https://api.github.com/users/foolip/orgs","repos_url":"https://api.github.com/users/foolip/repos","events_url":"https://api.github.com/users/foolip/events{/privacy}","received_events_url":"https://api.github.com/users/foolip/received_events","type":"User","site_admin":false},"body":"Heh, it started out like that, but I didn't like the method names I came up with. I'll call it \"analyze\" then.","created_at":"2018-04-18T12:28:52Z","updated_at":"2018-04-18T12:28:52Z","html_url":"https://github.com/foolip/ecosystem-infra-stats/pull/10#discussion_r182407800","pull_request_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/pulls/10","author_association":"OWNER","_links":{"self":{"href":"https://api.github.com/repos/foolip/ecosystem-infra-stats/pulls/comments/182407800"},"html":{"href":"https://github.com/foolip/ecosystem-infra-stats/pull/10#discussion_r182407800"},"pull_request":{"href":"https://api.github.com/repos/foolip/ecosystem-infra-stats/pulls/10"}},"in_reply_to_id":180871317},"pull_request":{"url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/pulls/10","id":179966171,"html_url":"https://github.com/foolip/ecosystem-infra-stats/pull/10","diff_url":"https://github.com/foolip/ecosystem-infra-stats/pull/10.diff","patch_url":"https://github.com/foolip/ecosystem-infra-stats/pull/10.patch","issue_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/issues/10","number":10,"state":"open","locked":false,"title":"Implement wpt.fyi run latency stats","user":{"login":"foolip","id":498917,"avatar_url":"https://avatars1.githubusercontent.com/u/498917?v=4","gravatar_id":"","url":"https://api.github.com/users/foolip","html_url":"https://github.com/foolip","followers_url":"https://api.github.com/users/foolip/followers","following_url":"https://api.github.com/users/foolip/following{/other_user}","gists_url":"https://api.github.com/users/foolip/gists{/gist_id}","starred_url":"https://api.github.com/users/foolip/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/foolip/subscriptions","organizations_url":"https://api.github.com/users/foolip/orgs","repos_url":"https://api.github.com/users/foolip/repos","events_url":"https://api.github.com/users/foolip/events{/privacy}","received_events_url":"https://api.github.com/users/foolip/received_events","type":"User","site_admin":false},"body":"Fixes https://github.com/foolip/ecosystem-infra-stats/issues/6.","created_at":"2018-04-06T14:33:57Z","updated_at":"2018-04-18T12:28:52Z","closed_at":null,"merged_at":null,"merge_commit_sha":"4b317e49cb99eb5698828678b247a553fd4a322f","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/pulls/10/commits","review_comments_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/pulls/10/comments","review_comment_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/pulls/comments{/number}","comments_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/issues/10/comments","statuses_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/statuses/126faef342e893df96e15d3a58fdeebe2658a360","head":{"label":"foolip:wpt-dashboard-stats","ref":"wpt-dashboard-stats","sha":"126faef342e893df96e15d3a58fdeebe2658a360","user":{"login":"foolip","id":498917,"avatar_url":"https://avatars1.githubusercontent.com/u/498917?v=4","gravatar_id":"","url":"https://api.github.com/users/foolip","html_url":"https://github.com/foolip","followers_url":"https://api.github.com/users/foolip/followers","following_url":"https://api.github.com/users/foolip/following{/other_user}","gists_url":"https://api.github.com/users/foolip/gists{/gist_id}","starred_url":"https://api.github.com/users/foolip/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/foolip/subscriptions","organizations_url":"https://api.github.com/users/foolip/orgs","repos_url":"https://api.github.com/users/foolip/repos","events_url":"https://api.github.com/users/foolip/events{/privacy}","received_events_url":"https://api.github.com/users/foolip/received_events","type":"User","site_admin":false},"repo":{"id":109427006,"name":"ecosystem-infra-stats","full_name":"foolip/ecosystem-infra-stats","owner":{"login":"foolip","id":498917,"avatar_url":"https://avatars1.githubusercontent.com/u/498917?v=4","gravatar_id":"","url":"https://api.github.com/users/foolip","html_url":"https://github.com/foolip","followers_url":"https://api.github.com/users/foolip/followers","following_url":"https://api.github.com/users/foolip/following{/other_user}","gists_url":"https://api.github.com/users/foolip/gists{/gist_id}","starred_url":"https://api.github.com/users/foolip/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/foolip/subscriptions","organizations_url":"https://api.github.com/users/foolip/orgs","repos_url":"https://api.github.com/users/foolip/repos","events_url":"https://api.github.com/users/foolip/events{/privacy}","received_events_url":"https://api.github.com/users/foolip/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/foolip/ecosystem-infra-stats","description":"Ecosystem Infra Stats","fork":false,"url":"https://api.github.com/repos/foolip/ecosystem-infra-stats","forks_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/forks","keys_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/keys{/key_id}","collaborators_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/teams","hooks_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/hooks","issue_events_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/issues/events{/number}","events_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/events","assignees_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/assignees{/user}","branches_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/branches{/branch}","tags_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/tags","blobs_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/git/refs{/sha}","trees_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/git/trees{/sha}","statuses_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/statuses/{sha}","languages_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/languages","stargazers_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/stargazers","contributors_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/contributors","subscribers_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/subscribers","subscription_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/subscription","commits_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/commits{/sha}","git_commits_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/git/commits{/sha}","comments_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/comments{/number}","issue_comment_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/issues/comments{/number}","contents_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/contents/{+path}","compare_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/compare/{base}...{head}","merges_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/merges","archive_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/downloads","issues_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/issues{/number}","pulls_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/pulls{/number}","milestones_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/milestones{/number}","notifications_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/labels{/name}","releases_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/releases{/id}","deployments_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/deployments","created_at":"2017-11-03T17:57:50Z","updated_at":"2018-04-16T19:45:23Z","pushed_at":"2018-04-18T12:24:44Z","git_url":"git://github.com/foolip/ecosystem-infra-stats.git","ssh_url":"git@github.com:foolip/ecosystem-infra-stats.git","clone_url":"https://github.com/foolip/ecosystem-infra-stats.git","svn_url":"https://github.com/foolip/ecosystem-infra-stats","homepage":"https://foolip.github.io/ecosystem-infra-stats/","size":1199,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":1,"mirror_url":null,"archived":false,"open_issues_count":4,"license":null,"forks":1,"open_issues":4,"watchers":0,"default_branch":"master"}},"base":{"label":"foolip:master","ref":"master","sha":"c31ecb0ef44fd020486a4daf30d227f96f86df56","user":{"login":"foolip","id":498917,"avatar_url":"https://avatars1.githubusercontent.com/u/498917?v=4","gravatar_id":"","url":"https://api.github.com/users/foolip","html_url":"https://github.com/foolip","followers_url":"https://api.github.com/users/foolip/followers","following_url":"https://api.github.com/users/foolip/following{/other_user}","gists_url":"https://api.github.com/users/foolip/gists{/gist_id}","starred_url":"https://api.github.com/users/foolip/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/foolip/subscriptions","organizations_url":"https://api.github.com/users/foolip/orgs","repos_url":"https://api.github.com/users/foolip/repos","events_url":"https://api.github.com/users/foolip/events{/privacy}","received_events_url":"https://api.github.com/users/foolip/received_events","type":"User","site_admin":false},"repo":{"id":109427006,"name":"ecosystem-infra-stats","full_name":"foolip/ecosystem-infra-stats","owner":{"login":"foolip","id":498917,"avatar_url":"https://avatars1.githubusercontent.com/u/498917?v=4","gravatar_id":"","url":"https://api.github.com/users/foolip","html_url":"https://github.com/foolip","followers_url":"https://api.github.com/users/foolip/followers","following_url":"https://api.github.com/users/foolip/following{/other_user}","gists_url":"https://api.github.com/users/foolip/gists{/gist_id}","starred_url":"https://api.github.com/users/foolip/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/foolip/subscriptions","organizations_url":"https://api.github.com/users/foolip/orgs","repos_url":"https://api.github.com/users/foolip/repos","events_url":"https://api.github.com/users/foolip/events{/privacy}","received_events_url":"https://api.github.com/users/foolip/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/foolip/ecosystem-infra-stats","description":"Ecosystem Infra Stats","fork":false,"url":"https://api.github.com/repos/foolip/ecosystem-infra-stats","forks_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/forks","keys_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/keys{/key_id}","collaborators_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/teams","hooks_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/hooks","issue_events_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/issues/events{/number}","events_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/events","assignees_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/assignees{/user}","branches_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/branches{/branch}","tags_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/tags","blobs_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/git/refs{/sha}","trees_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/git/trees{/sha}","statuses_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/statuses/{sha}","languages_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/languages","stargazers_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/stargazers","contributors_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/contributors","subscribers_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/subscribers","subscription_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/subscription","commits_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/commits{/sha}","git_commits_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/git/commits{/sha}","comments_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/comments{/number}","issue_comment_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/issues/comments{/number}","contents_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/contents/{+path}","compare_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/compare/{base}...{head}","merges_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/merges","archive_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/downloads","issues_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/issues{/number}","pulls_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/pulls{/number}","milestones_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/milestones{/number}","notifications_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/labels{/name}","releases_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/releases{/id}","deployments_url":"https://api.github.com/repos/foolip/ecosystem-infra-stats/deployments","created_at":"2017-11-03T17:57:50Z","updated_at":"2018-04-16T19:45:23Z","pushed_at":"2018-04-18T12:24:44Z","git_url":"git://github.com/foolip/ecosystem-infra-stats.git","ssh_url":"git@github.com:foolip/ecosystem-infra-stats.git","clone_url":"https://github.com/foolip/ecosystem-infra-stats.git","svn_url":"https://github.com/foolip/ecosystem-infra-stats","homepage":"https://foolip.github.io/ecosystem-infra-stats/","size":1199,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":1,"mirror_url":null,"archived":false,"open_issues_count":4,"license":null,"forks":1,"open_issues":4,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/foolip/ecosystem-infra-stats/pulls/10"},"html":{"href":"https://github.com/foolip/ecosystem-infra-stats/pull/10"},"issue":{"href":"https://api.github.com/repos/foolip/ecosystem-infra-stats/issues/10"},"comments":{"href":"https://api.github.com/repos/foolip/ecosystem-infra-stats/issues/10/comments"},"review_comments":{"href":"https://api.github.com/repos/foolip/ecosystem-infra-stats/pulls/10/comments"},"review_comment":{"href":"https://api.github.com/repos/foolip/ecosystem-infra-stats/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/foolip/ecosystem-infra-stats/pulls/10/commits"},"statuses":{"href":"https://api.github.com/repos/foolip/ecosystem-infra-stats/statuses/126faef342e893df96e15d3a58fdeebe2658a360"}},"author_association":"OWNER"}}
{"actor":{"display_login":"foolip"}}