{"action":"created","comment":{"url":"https://api.github.com/repos/pachyderm/pachyderm/pulls/comments/81639160","pull_request_review_id":2602489,"id":81639160,"diff_hunk":"@@ -341,3 +342,107 @@ func TestFruitStand(t *testing.T) {\n \trequire.NoError(t, c.DeleteRepo(\"filter\", false))\n \trequire.NoError(t, c.DeleteRepo(\"data\", false))\n }\n+\n+func TestScraper(t *testing.T) {\n+\tif testing.Short() {\n+\t\tt.Skip(\"Skipping integration tests in short mode\")\n+\t}\n+\tc := getPachClient(t)\n+\n+\t// Check that scraper pipeline manifest appears in the docs (before we create a new repo or anything)\n+\treadme, err := ioutil.ReadFile(\"../../../doc/examples/scraper/README.md\")\n+\trequire.NoError(t, err)\n+\tinputPipelineManifest := `\n+{\n+  \"pipeline\": {\n+    \"name\": \"scraper”\n+  },\n+  \"transform\": {\n+    \"cmd\": [ \"wget\",\n+        \"--recursive\",\n+        \"--level\", \"1\",\n+        \"--accept\", \"jpg,jpeg,png,gif,bmp\",\n+        \"--page-requisites\",\n+        \"--adjust-extension\",\n+        \"--span-hosts\",\n+        \"--no-check-certificate\",\n+        \"--timestamping\",\n+        \"--directory-prefix\",\n+        \"/pfs/out\",\n+        \"--input-file\", \"/pfs/urls/urls\"\n+    ],\n+    \"acceptReturnCode\": [4,5,6,7,8]\n+  },\n+  \"parallelism\": \"1\",\n+  \"inputs\": [\n+    {\n+      \"repo\": {\n+        \"name\": \"urls\"\n+      }\n+    }\n+  ]\n+}\n+`\n+\trequire.Equal(t, true, strings.Contains(string(readme), inputPipelineManifest))\n+\n+\t// create \"urls\" repo and check that it exists\n+\tcmd := exec.Command(\"pachctl\", \"create-repo\", \"urls\")\n+\trequire.NoError(t, cmd.Run())\n+\tvar pfsRepos []*pfsclient.RepoInfo\n+\tpfsRepos, err = c.ListRepo([]string{})\n+\trequire.NoError(t, err)\n+\trequire.EqualOneOf(t, toRepoNames(pfsRepos), \"urls\")\n+\n+\t// Create first commit in repo\n+\tvar output []byte\n+\toutput, err = exec.Command(\"pachctl\", \"start-commit\", \"urls\", \"master\").Output()\n+\trequire.NoError(t, err)\n+\trequire.Equal(t, \"master/0\\n\", string(output))\n+\t// Check that it's there\n+\tvar commitInfos []*pfsclient.CommitInfo\n+\tcommitInfos, err = c.ListCommit(\n+\t\t// fromCommits (only use commits to the 'urls' repo)\n+\t\t[]*pfsclient.Commit{{\n+\t\t\tRepo: &pfsclient.Repo{\"urls\"},\n+\t\t}},\n+\t\tnil, // provenance (any provenance)\n+\t\tclient.CommitTypeWrite, // Commit hasn't finished, so we want \"write\"\n+\t\tpfsclient.CommitStatus_NORMAL,\n+\t\ttrue)\n+\trequire.NoError(t, err)\n+\trequire.Equal(t, 1, len(commitInfos))\n+\trequire.Equal(t, \"urls\", commitInfos[0].Commit.Repo.Name)\n+\trequire.Equal(t, \"master/0\", commitInfos[0].Commit.ID)\n+\n+\t// Example input takes ~10 minutes to scrape, so just use two URLs here\n+\tcmd = exec.Command(\"pachctl\", \"put-file\", \"urls\", \"master/0\", \"urls\")\n+\tcmd.Stdin = bytes.NewBuffer([]byte(\"www.google.com\\nwww.example.com\\n\"))\n+\trequire.NoError(t, cmd.Run())\n+\n+\t// finish commit\n+\trequire.NoError(t, exec.Command(\"pachctl\", \"finish-commit\", \"urls\", \"master/0\").Run())\n+\tb := &bytes.Buffer{}\n+\trequire.NoError(t, c.GetFile(\"urls\", \"master/0\", \"urls\", 0, 0, \"\", false, &pfsclient.Shard{}, b))\n+\n+\t// Create pipeline\n+\tcmd = exec.Command(\"pachctl\", \"create-pipeline\", \"-f\", \"-\")\n+\tcmd.Stdin, err = os.Open(\"../../../doc/examples/scraper/scraper.json\")\n+\trequire.NoError(t, err)\n+\trequire.NoError(t, cmd.Run())\n+\tc.FlushCommit([]*pfsclient.Commit{commitInfos[0].Commit}, nil) // Wait until the URLs have been scraped","path":"src/server/examples/examples_test.go","position":131,"original_position":131,"commit_id":"74c814fe1ab7bdf0f87e2c6dd7b24983bd408a36","original_commit_id":"74c814fe1ab7bdf0f87e2c6dd7b24983bd408a36","user":{"login":"sjezewski","id":597953,"avatar_url":"https://avatars.githubusercontent.com/u/597953?v=3","gravatar_id":"","url":"https://api.github.com/users/sjezewski","html_url":"https://github.com/sjezewski","followers_url":"https://api.github.com/users/sjezewski/followers","following_url":"https://api.github.com/users/sjezewski/following{/other_user}","gists_url":"https://api.github.com/users/sjezewski/gists{/gist_id}","starred_url":"https://api.github.com/users/sjezewski/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sjezewski/subscriptions","organizations_url":"https://api.github.com/users/sjezewski/orgs","repos_url":"https://api.github.com/users/sjezewski/repos","events_url":"https://api.github.com/users/sjezewski/events{/privacy}","received_events_url":"https://api.github.com/users/sjezewski/received_events","type":"User","site_admin":false},"body":"Even if you're not using the returned commitInfos, may as well make sure `FlushCommit` didn't err","created_at":"2016-10-03T21:01:25Z","updated_at":"2016-10-03T21:03:01Z","html_url":"https://github.com/pachyderm/pachyderm/pull/916#discussion_r81639160","pull_request_url":"https://api.github.com/repos/pachyderm/pachyderm/pulls/916","_links":{"self":{"href":"https://api.github.com/repos/pachyderm/pachyderm/pulls/comments/81639160"},"html":{"href":"https://github.com/pachyderm/pachyderm/pull/916#discussion_r81639160"},"pull_request":{"href":"https://api.github.com/repos/pachyderm/pachyderm/pulls/916"}}},"pull_request":{"url":"https://api.github.com/repos/pachyderm/pachyderm/pulls/916","id":87374377,"html_url":"https://github.com/pachyderm/pachyderm/pull/916","diff_url":"https://github.com/pachyderm/pachyderm/pull/916.diff","patch_url":"https://github.com/pachyderm/pachyderm/pull/916.patch","issue_url":"https://api.github.com/repos/pachyderm/pachyderm/issues/916","number":916,"state":"open","locked":false,"title":"Add integration test for scraper demo","user":{"login":"msteffen","id":235674,"avatar_url":"https://avatars.githubusercontent.com/u/235674?v=3","gravatar_id":"","url":"https://api.github.com/users/msteffen","html_url":"https://github.com/msteffen","followers_url":"https://api.github.com/users/msteffen/followers","following_url":"https://api.github.com/users/msteffen/following{/other_user}","gists_url":"https://api.github.com/users/msteffen/gists{/gist_id}","starred_url":"https://api.github.com/users/msteffen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/msteffen/subscriptions","organizations_url":"https://api.github.com/users/msteffen/orgs","repos_url":"https://api.github.com/users/msteffen/repos","events_url":"https://api.github.com/users/msteffen/events{/privacy}","received_events_url":"https://api.github.com/users/msteffen/received_events","type":"User","site_admin":false},"body":"I also made a few minor corrections to the scraper demo doc","created_at":"2016-09-29T19:07:46Z","updated_at":"2016-10-03T21:03:01Z","closed_at":null,"merged_at":null,"merge_commit_sha":"0fad5f3eca93fdced90c111e1f4d95fe3527d416","assignee":{"login":"sjezewski","id":597953,"avatar_url":"https://avatars.githubusercontent.com/u/597953?v=3","gravatar_id":"","url":"https://api.github.com/users/sjezewski","html_url":"https://github.com/sjezewski","followers_url":"https://api.github.com/users/sjezewski/followers","following_url":"https://api.github.com/users/sjezewski/following{/other_user}","gists_url":"https://api.github.com/users/sjezewski/gists{/gist_id}","starred_url":"https://api.github.com/users/sjezewski/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sjezewski/subscriptions","organizations_url":"https://api.github.com/users/sjezewski/orgs","repos_url":"https://api.github.com/users/sjezewski/repos","events_url":"https://api.github.com/users/sjezewski/events{/privacy}","received_events_url":"https://api.github.com/users/sjezewski/received_events","type":"User","site_admin":false},"assignees":[{"login":"sjezewski","id":597953,"avatar_url":"https://avatars.githubusercontent.com/u/597953?v=3","gravatar_id":"","url":"https://api.github.com/users/sjezewski","html_url":"https://github.com/sjezewski","followers_url":"https://api.github.com/users/sjezewski/followers","following_url":"https://api.github.com/users/sjezewski/following{/other_user}","gists_url":"https://api.github.com/users/sjezewski/gists{/gist_id}","starred_url":"https://api.github.com/users/sjezewski/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sjezewski/subscriptions","organizations_url":"https://api.github.com/users/sjezewski/orgs","repos_url":"https://api.github.com/users/sjezewski/repos","events_url":"https://api.github.com/users/sjezewski/events{/privacy}","received_events_url":"https://api.github.com/users/sjezewski/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/pachyderm/pachyderm/pulls/916/commits","review_comments_url":"https://api.github.com/repos/pachyderm/pachyderm/pulls/916/comments","review_comment_url":"https://api.github.com/repos/pachyderm/pachyderm/pulls/comments{/number}","comments_url":"https://api.github.com/repos/pachyderm/pachyderm/issues/916/comments","statuses_url":"https://api.github.com/repos/pachyderm/pachyderm/statuses/74c814fe1ab7bdf0f87e2c6dd7b24983bd408a36","head":{"label":"pachyderm:scraper_test","ref":"scraper_test","sha":"74c814fe1ab7bdf0f87e2c6dd7b24983bd408a36","user":{"login":"pachyderm","id":10432478,"avatar_url":"https://avatars.githubusercontent.com/u/10432478?v=3","gravatar_id":"","url":"https://api.github.com/users/pachyderm","html_url":"https://github.com/pachyderm","followers_url":"https://api.github.com/users/pachyderm/followers","following_url":"https://api.github.com/users/pachyderm/following{/other_user}","gists_url":"https://api.github.com/users/pachyderm/gists{/gist_id}","starred_url":"https://api.github.com/users/pachyderm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pachyderm/subscriptions","organizations_url":"https://api.github.com/users/pachyderm/orgs","repos_url":"https://api.github.com/users/pachyderm/repos","events_url":"https://api.github.com/users/pachyderm/events{/privacy}","received_events_url":"https://api.github.com/users/pachyderm/received_events","type":"Organization","site_admin":false},"repo":{"id":23653453,"name":"pachyderm","full_name":"pachyderm/pachyderm","owner":{"login":"pachyderm","id":10432478,"avatar_url":"https://avatars.githubusercontent.com/u/10432478?v=3","gravatar_id":"","url":"https://api.github.com/users/pachyderm","html_url":"https://github.com/pachyderm","followers_url":"https://api.github.com/users/pachyderm/followers","following_url":"https://api.github.com/users/pachyderm/following{/other_user}","gists_url":"https://api.github.com/users/pachyderm/gists{/gist_id}","starred_url":"https://api.github.com/users/pachyderm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pachyderm/subscriptions","organizations_url":"https://api.github.com/users/pachyderm/orgs","repos_url":"https://api.github.com/users/pachyderm/repos","events_url":"https://api.github.com/users/pachyderm/events{/privacy}","received_events_url":"https://api.github.com/users/pachyderm/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/pachyderm/pachyderm","description":"Containerized Data Analytics","fork":false,"url":"https://api.github.com/repos/pachyderm/pachyderm","forks_url":"https://api.github.com/repos/pachyderm/pachyderm/forks","keys_url":"https://api.github.com/repos/pachyderm/pachyderm/keys{/key_id}","collaborators_url":"https://api.github.com/repos/pachyderm/pachyderm/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/pachyderm/pachyderm/teams","hooks_url":"https://api.github.com/repos/pachyderm/pachyderm/hooks","issue_events_url":"https://api.github.com/repos/pachyderm/pachyderm/issues/events{/number}","events_url":"https://api.github.com/repos/pachyderm/pachyderm/events","assignees_url":"https://api.github.com/repos/pachyderm/pachyderm/assignees{/user}","branches_url":"https://api.github.com/repos/pachyderm/pachyderm/branches{/branch}","tags_url":"https://api.github.com/repos/pachyderm/pachyderm/tags","blobs_url":"https://api.github.com/repos/pachyderm/pachyderm/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/pachyderm/pachyderm/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/pachyderm/pachyderm/git/refs{/sha}","trees_url":"https://api.github.com/repos/pachyderm/pachyderm/git/trees{/sha}","statuses_url":"https://api.github.com/repos/pachyderm/pachyderm/statuses/{sha}","languages_url":"https://api.github.com/repos/pachyderm/pachyderm/languages","stargazers_url":"https://api.github.com/repos/pachyderm/pachyderm/stargazers","contributors_url":"https://api.github.com/repos/pachyderm/pachyderm/contributors","subscribers_url":"https://api.github.com/repos/pachyderm/pachyderm/subscribers","subscription_url":"https://api.github.com/repos/pachyderm/pachyderm/subscription","commits_url":"https://api.github.com/repos/pachyderm/pachyderm/commits{/sha}","git_commits_url":"https://api.github.com/repos/pachyderm/pachyderm/git/commits{/sha}","comments_url":"https://api.github.com/repos/pachyderm/pachyderm/comments{/number}","issue_comment_url":"https://api.github.com/repos/pachyderm/pachyderm/issues/comments{/number}","contents_url":"https://api.github.com/repos/pachyderm/pachyderm/contents/{+path}","compare_url":"https://api.github.com/repos/pachyderm/pachyderm/compare/{base}...{head}","merges_url":"https://api.github.com/repos/pachyderm/pachyderm/merges","archive_url":"https://api.github.com/repos/pachyderm/pachyderm/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/pachyderm/pachyderm/downloads","issues_url":"https://api.github.com/repos/pachyderm/pachyderm/issues{/number}","pulls_url":"https://api.github.com/repos/pachyderm/pachyderm/pulls{/number}","milestones_url":"https://api.github.com/repos/pachyderm/pachyderm/milestones{/number}","notifications_url":"https://api.github.com/repos/pachyderm/pachyderm/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/pachyderm/pachyderm/labels{/name}","releases_url":"https://api.github.com/repos/pachyderm/pachyderm/releases{/id}","deployments_url":"https://api.github.com/repos/pachyderm/pachyderm/deployments","created_at":"2014-09-04T07:50:02Z","updated_at":"2016-10-03T15:33:19Z","pushed_at":"2016-10-02T17:00:20Z","git_url":"git://github.com/pachyderm/pachyderm.git","ssh_url":"git@github.com:pachyderm/pachyderm.git","clone_url":"https://github.com/pachyderm/pachyderm.git","svn_url":"https://github.com/pachyderm/pachyderm","homepage":"http://pachyderm.io","size":35131,"stargazers_count":1312,"watchers_count":1312,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":131,"mirror_url":null,"open_issues_count":150,"forks":131,"open_issues":150,"watchers":1312,"default_branch":"master"}},"base":{"label":"pachyderm:master","ref":"master","sha":"7242ed1fa94449c05a8645e7bb6d2616da56926e","user":{"login":"pachyderm","id":10432478,"avatar_url":"https://avatars.githubusercontent.com/u/10432478?v=3","gravatar_id":"","url":"https://api.github.com/users/pachyderm","html_url":"https://github.com/pachyderm","followers_url":"https://api.github.com/users/pachyderm/followers","following_url":"https://api.github.com/users/pachyderm/following{/other_user}","gists_url":"https://api.github.com/users/pachyderm/gists{/gist_id}","starred_url":"https://api.github.com/users/pachyderm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pachyderm/subscriptions","organizations_url":"https://api.github.com/users/pachyderm/orgs","repos_url":"https://api.github.com/users/pachyderm/repos","events_url":"https://api.github.com/users/pachyderm/events{/privacy}","received_events_url":"https://api.github.com/users/pachyderm/received_events","type":"Organization","site_admin":false},"repo":{"id":23653453,"name":"pachyderm","full_name":"pachyderm/pachyderm","owner":{"login":"pachyderm","id":10432478,"avatar_url":"https://avatars.githubusercontent.com/u/10432478?v=3","gravatar_id":"","url":"https://api.github.com/users/pachyderm","html_url":"https://github.com/pachyderm","followers_url":"https://api.github.com/users/pachyderm/followers","following_url":"https://api.github.com/users/pachyderm/following{/other_user}","gists_url":"https://api.github.com/users/pachyderm/gists{/gist_id}","starred_url":"https://api.github.com/users/pachyderm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pachyderm/subscriptions","organizations_url":"https://api.github.com/users/pachyderm/orgs","repos_url":"https://api.github.com/users/pachyderm/repos","events_url":"https://api.github.com/users/pachyderm/events{/privacy}","received_events_url":"https://api.github.com/users/pachyderm/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/pachyderm/pachyderm","description":"Containerized Data Analytics","fork":false,"url":"https://api.github.com/repos/pachyderm/pachyderm","forks_url":"https://api.github.com/repos/pachyderm/pachyderm/forks","keys_url":"https://api.github.com/repos/pachyderm/pachyderm/keys{/key_id}","collaborators_url":"https://api.github.com/repos/pachyderm/pachyderm/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/pachyderm/pachyderm/teams","hooks_url":"https://api.github.com/repos/pachyderm/pachyderm/hooks","issue_events_url":"https://api.github.com/repos/pachyderm/pachyderm/issues/events{/number}","events_url":"https://api.github.com/repos/pachyderm/pachyderm/events","assignees_url":"https://api.github.com/repos/pachyderm/pachyderm/assignees{/user}","branches_url":"https://api.github.com/repos/pachyderm/pachyderm/branches{/branch}","tags_url":"https://api.github.com/repos/pachyderm/pachyderm/tags","blobs_url":"https://api.github.com/repos/pachyderm/pachyderm/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/pachyderm/pachyderm/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/pachyderm/pachyderm/git/refs{/sha}","trees_url":"https://api.github.com/repos/pachyderm/pachyderm/git/trees{/sha}","statuses_url":"https://api.github.com/repos/pachyderm/pachyderm/statuses/{sha}","languages_url":"https://api.github.com/repos/pachyderm/pachyderm/languages","stargazers_url":"https://api.github.com/repos/pachyderm/pachyderm/stargazers","contributors_url":"https://api.github.com/repos/pachyderm/pachyderm/contributors","subscribers_url":"https://api.github.com/repos/pachyderm/pachyderm/subscribers","subscription_url":"https://api.github.com/repos/pachyderm/pachyderm/subscription","commits_url":"https://api.github.com/repos/pachyderm/pachyderm/commits{/sha}","git_commits_url":"https://api.github.com/repos/pachyderm/pachyderm/git/commits{/sha}","comments_url":"https://api.github.com/repos/pachyderm/pachyderm/comments{/number}","issue_comment_url":"https://api.github.com/repos/pachyderm/pachyderm/issues/comments{/number}","contents_url":"https://api.github.com/repos/pachyderm/pachyderm/contents/{+path}","compare_url":"https://api.github.com/repos/pachyderm/pachyderm/compare/{base}...{head}","merges_url":"https://api.github.com/repos/pachyderm/pachyderm/merges","archive_url":"https://api.github.com/repos/pachyderm/pachyderm/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/pachyderm/pachyderm/downloads","issues_url":"https://api.github.com/repos/pachyderm/pachyderm/issues{/number}","pulls_url":"https://api.github.com/repos/pachyderm/pachyderm/pulls{/number}","milestones_url":"https://api.github.com/repos/pachyderm/pachyderm/milestones{/number}","notifications_url":"https://api.github.com/repos/pachyderm/pachyderm/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/pachyderm/pachyderm/labels{/name}","releases_url":"https://api.github.com/repos/pachyderm/pachyderm/releases{/id}","deployments_url":"https://api.github.com/repos/pachyderm/pachyderm/deployments","created_at":"2014-09-04T07:50:02Z","updated_at":"2016-10-03T15:33:19Z","pushed_at":"2016-10-02T17:00:20Z","git_url":"git://github.com/pachyderm/pachyderm.git","ssh_url":"git@github.com:pachyderm/pachyderm.git","clone_url":"https://github.com/pachyderm/pachyderm.git","svn_url":"https://github.com/pachyderm/pachyderm","homepage":"http://pachyderm.io","size":35131,"stargazers_count":1312,"watchers_count":1312,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":131,"mirror_url":null,"open_issues_count":150,"forks":131,"open_issues":150,"watchers":1312,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/pachyderm/pachyderm/pulls/916"},"html":{"href":"https://github.com/pachyderm/pachyderm/pull/916"},"issue":{"href":"https://api.github.com/repos/pachyderm/pachyderm/issues/916"},"comments":{"href":"https://api.github.com/repos/pachyderm/pachyderm/issues/916/comments"},"review_comments":{"href":"https://api.github.com/repos/pachyderm/pachyderm/pulls/916/comments"},"review_comment":{"href":"https://api.github.com/repos/pachyderm/pachyderm/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/pachyderm/pachyderm/pulls/916/commits"},"statuses":{"href":"https://api.github.com/repos/pachyderm/pachyderm/statuses/74c814fe1ab7bdf0f87e2c6dd7b24983bd408a36"}}}}
{"actor":{"display_login":"sjezewski"}}