{"action":"created","comment":{"url":"https://api.github.com/repos/dask/distributed/pulls/comments/55100674","id":55100674,"diff_hunk":"@@ -393,3 +394,91 @@ def test_read_csv_sync(loop):\n             assert list(df.columns) == ['name', 'amount', 'id']\n             f = e.compute(df.amount.sum())\n             assert f.result() == (100 + 200 + 300 + 400 + 500 + 600)\n+\n+def test_avro_body(s3):\n+    fastavro = pytest.importorskip('fastavro')\n+    fn = 'distributed-test/test/data/avro/2.avro'\n+    byte = s3.cat(fn)\n+    av = fastavro.reader(s3.open(fn))\n+    header = av._header\n+    records = list(av)\n+    \n+    schema = header['meta']['avro.schema'].decode()\n+    schema = json.loads(schema)\n+    assert ['amount', 'name'] == sorted(f['name'] for f in schema['fields'])\n+    sync = header['sync']\n+    assert len(sync) == 16\n+    bit = byte[byte.index(sync)+len(sync):]\n+    assert avro_body(bit, header) == records\n+    bit = byte[byte.index(sync)+len(sync):-len(sync)]\n+    assert avro_body(bit, header) == records\n+    assert avro_body(b\"\", header) == []","path":"distributed/tests/test_s3.py","position":32,"original_position":32,"commit_id":"da343f3b0c2430223a0bfc017dc6d044403b9bef","original_commit_id":"da343f3b0c2430223a0bfc017dc6d044403b9bef","user":{"login":"mrocklin","id":306380,"avatar_url":"https://avatars.githubusercontent.com/u/306380?v=3","gravatar_id":"","url":"https://api.github.com/users/mrocklin","html_url":"https://github.com/mrocklin","followers_url":"https://api.github.com/users/mrocklin/followers","following_url":"https://api.github.com/users/mrocklin/following{/other_user}","gists_url":"https://api.github.com/users/mrocklin/gists{/gist_id}","starred_url":"https://api.github.com/users/mrocklin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrocklin/subscriptions","organizations_url":"https://api.github.com/users/mrocklin/orgs","repos_url":"https://api.github.com/users/mrocklin/repos","events_url":"https://api.github.com/users/mrocklin/events{/privacy}","received_events_url":"https://api.github.com/users/mrocklin/received_events","type":"User","site_admin":false},"body":"Ideally we would be able to test this function without relying on S3.  Perhaps now would be a good time to think about refactoring out formats?  We're not particularly time constrained on the avro work (no one is actively asking for it) so the choice between copy-paste and refactor might change here (we may want to refactor.)","created_at":"2016-03-04T22:44:46Z","updated_at":"2016-03-04T22:44:46Z","html_url":"https://github.com/dask/distributed/pull/167#discussion_r55100674","pull_request_url":"https://api.github.com/repos/dask/distributed/pulls/167","_links":{"self":{"href":"https://api.github.com/repos/dask/distributed/pulls/comments/55100674"},"html":{"href":"https://github.com/dask/distributed/pull/167#discussion_r55100674"},"pull_request":{"href":"https://api.github.com/repos/dask/distributed/pulls/167"}}},"pull_request":{"url":"https://api.github.com/repos/dask/distributed/pulls/167","id":61775713,"html_url":"https://github.com/dask/distributed/pull/167","diff_url":"https://github.com/dask/distributed/pull/167.diff","patch_url":"https://github.com/dask/distributed/pull/167.patch","issue_url":"https://api.github.com/repos/dask/distributed/issues/167","number":167,"state":"open","locked":false,"title":"Add back S3 avro functions, with tests","user":{"login":"martindurant","id":6042212,"avatar_url":"https://avatars.githubusercontent.com/u/6042212?v=3","gravatar_id":"","url":"https://api.github.com/users/martindurant","html_url":"https://github.com/martindurant","followers_url":"https://api.github.com/users/martindurant/followers","following_url":"https://api.github.com/users/martindurant/following{/other_user}","gists_url":"https://api.github.com/users/martindurant/gists{/gist_id}","starred_url":"https://api.github.com/users/martindurant/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/martindurant/subscriptions","organizations_url":"https://api.github.com/users/martindurant/orgs","repos_url":"https://api.github.com/users/martindurant/repos","events_url":"https://api.github.com/users/martindurant/events{/privacy}","received_events_url":"https://api.github.com/users/martindurant/received_events","type":"User","site_admin":false},"body":"Tests are largely modelled on read_csv, but with some additions for\r\nthe avro helper functions.","created_at":"2016-03-04T21:42:04Z","updated_at":"2016-03-04T22:44:46Z","closed_at":null,"merged_at":null,"merge_commit_sha":"6f41cb978a585e1d76cd2cebebcbe3c2276d527a","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/dask/distributed/pulls/167/commits","review_comments_url":"https://api.github.com/repos/dask/distributed/pulls/167/comments","review_comment_url":"https://api.github.com/repos/dask/distributed/pulls/comments{/number}","comments_url":"https://api.github.com/repos/dask/distributed/issues/167/comments","statuses_url":"https://api.github.com/repos/dask/distributed/statuses/da343f3b0c2430223a0bfc017dc6d044403b9bef","head":{"label":"martindurant:s3-avro","ref":"s3-avro","sha":"da343f3b0c2430223a0bfc017dc6d044403b9bef","user":{"login":"martindurant","id":6042212,"avatar_url":"https://avatars.githubusercontent.com/u/6042212?v=3","gravatar_id":"","url":"https://api.github.com/users/martindurant","html_url":"https://github.com/martindurant","followers_url":"https://api.github.com/users/martindurant/followers","following_url":"https://api.github.com/users/martindurant/following{/other_user}","gists_url":"https://api.github.com/users/martindurant/gists{/gist_id}","starred_url":"https://api.github.com/users/martindurant/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/martindurant/subscriptions","organizations_url":"https://api.github.com/users/martindurant/orgs","repos_url":"https://api.github.com/users/martindurant/repos","events_url":"https://api.github.com/users/martindurant/events{/privacy}","received_events_url":"https://api.github.com/users/martindurant/received_events","type":"User","site_admin":false},"repo":{"id":49908259,"name":"distributed","full_name":"martindurant/distributed","owner":{"login":"martindurant","id":6042212,"avatar_url":"https://avatars.githubusercontent.com/u/6042212?v=3","gravatar_id":"","url":"https://api.github.com/users/martindurant","html_url":"https://github.com/martindurant","followers_url":"https://api.github.com/users/martindurant/followers","following_url":"https://api.github.com/users/martindurant/following{/other_user}","gists_url":"https://api.github.com/users/martindurant/gists{/gist_id}","starred_url":"https://api.github.com/users/martindurant/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/martindurant/subscriptions","organizations_url":"https://api.github.com/users/martindurant/orgs","repos_url":"https://api.github.com/users/martindurant/repos","events_url":"https://api.github.com/users/martindurant/events{/privacy}","received_events_url":"https://api.github.com/users/martindurant/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/martindurant/distributed","description":"","fork":true,"url":"https://api.github.com/repos/martindurant/distributed","forks_url":"https://api.github.com/repos/martindurant/distributed/forks","keys_url":"https://api.github.com/repos/martindurant/distributed/keys{/key_id}","collaborators_url":"https://api.github.com/repos/martindurant/distributed/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/martindurant/distributed/teams","hooks_url":"https://api.github.com/repos/martindurant/distributed/hooks","issue_events_url":"https://api.github.com/repos/martindurant/distributed/issues/events{/number}","events_url":"https://api.github.com/repos/martindurant/distributed/events","assignees_url":"https://api.github.com/repos/martindurant/distributed/assignees{/user}","branches_url":"https://api.github.com/repos/martindurant/distributed/branches{/branch}","tags_url":"https://api.github.com/repos/martindurant/distributed/tags","blobs_url":"https://api.github.com/repos/martindurant/distributed/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/martindurant/distributed/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/martindurant/distributed/git/refs{/sha}","trees_url":"https://api.github.com/repos/martindurant/distributed/git/trees{/sha}","statuses_url":"https://api.github.com/repos/martindurant/distributed/statuses/{sha}","languages_url":"https://api.github.com/repos/martindurant/distributed/languages","stargazers_url":"https://api.github.com/repos/martindurant/distributed/stargazers","contributors_url":"https://api.github.com/repos/martindurant/distributed/contributors","subscribers_url":"https://api.github.com/repos/martindurant/distributed/subscribers","subscription_url":"https://api.github.com/repos/martindurant/distributed/subscription","commits_url":"https://api.github.com/repos/martindurant/distributed/commits{/sha}","git_commits_url":"https://api.github.com/repos/martindurant/distributed/git/commits{/sha}","comments_url":"https://api.github.com/repos/martindurant/distributed/comments{/number}","issue_comment_url":"https://api.github.com/repos/martindurant/distributed/issues/comments{/number}","contents_url":"https://api.github.com/repos/martindurant/distributed/contents/{+path}","compare_url":"https://api.github.com/repos/martindurant/distributed/compare/{base}...{head}","merges_url":"https://api.github.com/repos/martindurant/distributed/merges","archive_url":"https://api.github.com/repos/martindurant/distributed/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/martindurant/distributed/downloads","issues_url":"https://api.github.com/repos/martindurant/distributed/issues{/number}","pulls_url":"https://api.github.com/repos/martindurant/distributed/pulls{/number}","milestones_url":"https://api.github.com/repos/martindurant/distributed/milestones{/number}","notifications_url":"https://api.github.com/repos/martindurant/distributed/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/martindurant/distributed/labels{/name}","releases_url":"https://api.github.com/repos/martindurant/distributed/releases{/id}","deployments_url":"https://api.github.com/repos/martindurant/distributed/deployments","created_at":"2016-01-18T21:59:13Z","updated_at":"2016-01-18T21:59:14Z","pushed_at":"2016-03-04T21:41:30Z","git_url":"git://github.com/martindurant/distributed.git","ssh_url":"git@github.com:martindurant/distributed.git","clone_url":"https://github.com/martindurant/distributed.git","svn_url":"https://github.com/martindurant/distributed","homepage":null,"size":1816,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"dask:master","ref":"master","sha":"c5ff562a2e447922e7e224d9f0d176c7368100e6","user":{"login":"dask","id":17131925,"avatar_url":"https://avatars.githubusercontent.com/u/17131925?v=3","gravatar_id":"","url":"https://api.github.com/users/dask","html_url":"https://github.com/dask","followers_url":"https://api.github.com/users/dask/followers","following_url":"https://api.github.com/users/dask/following{/other_user}","gists_url":"https://api.github.com/users/dask/gists{/gist_id}","starred_url":"https://api.github.com/users/dask/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dask/subscriptions","organizations_url":"https://api.github.com/users/dask/orgs","repos_url":"https://api.github.com/users/dask/repos","events_url":"https://api.github.com/users/dask/events{/privacy}","received_events_url":"https://api.github.com/users/dask/received_events","type":"Organization","site_admin":false},"repo":{"id":42408083,"name":"distributed","full_name":"dask/distributed","owner":{"login":"dask","id":17131925,"avatar_url":"https://avatars.githubusercontent.com/u/17131925?v=3","gravatar_id":"","url":"https://api.github.com/users/dask","html_url":"https://github.com/dask","followers_url":"https://api.github.com/users/dask/followers","following_url":"https://api.github.com/users/dask/following{/other_user}","gists_url":"https://api.github.com/users/dask/gists{/gist_id}","starred_url":"https://api.github.com/users/dask/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dask/subscriptions","organizations_url":"https://api.github.com/users/dask/orgs","repos_url":"https://api.github.com/users/dask/repos","events_url":"https://api.github.com/users/dask/events{/privacy}","received_events_url":"https://api.github.com/users/dask/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/dask/distributed","description":"A library for distributed computation in Python","fork":false,"url":"https://api.github.com/repos/dask/distributed","forks_url":"https://api.github.com/repos/dask/distributed/forks","keys_url":"https://api.github.com/repos/dask/distributed/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dask/distributed/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dask/distributed/teams","hooks_url":"https://api.github.com/repos/dask/distributed/hooks","issue_events_url":"https://api.github.com/repos/dask/distributed/issues/events{/number}","events_url":"https://api.github.com/repos/dask/distributed/events","assignees_url":"https://api.github.com/repos/dask/distributed/assignees{/user}","branches_url":"https://api.github.com/repos/dask/distributed/branches{/branch}","tags_url":"https://api.github.com/repos/dask/distributed/tags","blobs_url":"https://api.github.com/repos/dask/distributed/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dask/distributed/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dask/distributed/git/refs{/sha}","trees_url":"https://api.github.com/repos/dask/distributed/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dask/distributed/statuses/{sha}","languages_url":"https://api.github.com/repos/dask/distributed/languages","stargazers_url":"https://api.github.com/repos/dask/distributed/stargazers","contributors_url":"https://api.github.com/repos/dask/distributed/contributors","subscribers_url":"https://api.github.com/repos/dask/distributed/subscribers","subscription_url":"https://api.github.com/repos/dask/distributed/subscription","commits_url":"https://api.github.com/repos/dask/distributed/commits{/sha}","git_commits_url":"https://api.github.com/repos/dask/distributed/git/commits{/sha}","comments_url":"https://api.github.com/repos/dask/distributed/comments{/number}","issue_comment_url":"https://api.github.com/repos/dask/distributed/issues/comments{/number}","contents_url":"https://api.github.com/repos/dask/distributed/contents/{+path}","compare_url":"https://api.github.com/repos/dask/distributed/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dask/distributed/merges","archive_url":"https://api.github.com/repos/dask/distributed/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dask/distributed/downloads","issues_url":"https://api.github.com/repos/dask/distributed/issues{/number}","pulls_url":"https://api.github.com/repos/dask/distributed/pulls{/number}","milestones_url":"https://api.github.com/repos/dask/distributed/milestones{/number}","notifications_url":"https://api.github.com/repos/dask/distributed/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dask/distributed/labels{/name}","releases_url":"https://api.github.com/repos/dask/distributed/releases{/id}","deployments_url":"https://api.github.com/repos/dask/distributed/deployments","created_at":"2015-09-13T18:42:29Z","updated_at":"2016-03-02T15:29:41Z","pushed_at":"2016-03-04T22:31:25Z","git_url":"git://github.com/dask/distributed.git","ssh_url":"git@github.com:dask/distributed.git","clone_url":"https://github.com/dask/distributed.git","svn_url":"https://github.com/dask/distributed","homepage":"http://distributed.readthedocs.org/en/latest/","size":1919,"stargazers_count":58,"watchers_count":58,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":19,"mirror_url":null,"open_issues_count":25,"forks":19,"open_issues":25,"watchers":58,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/dask/distributed/pulls/167"},"html":{"href":"https://github.com/dask/distributed/pull/167"},"issue":{"href":"https://api.github.com/repos/dask/distributed/issues/167"},"comments":{"href":"https://api.github.com/repos/dask/distributed/issues/167/comments"},"review_comments":{"href":"https://api.github.com/repos/dask/distributed/pulls/167/comments"},"review_comment":{"href":"https://api.github.com/repos/dask/distributed/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/dask/distributed/pulls/167/commits"},"statuses":{"href":"https://api.github.com/repos/dask/distributed/statuses/da343f3b0c2430223a0bfc017dc6d044403b9bef"}}}}