{"action":"created","comment":{"url":"https://api.github.com/repos/apache/incubator-beam/pulls/comments/91356811","pull_request_review_id":11866086,"id":91356811,"diff_hunk":"@@ -0,0 +1,383 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one\n+ * or more contributor license agreements.  See the NOTICE file\n+ * distributed with this work for additional information\n+ * regarding copyright ownership.  The ASF licenses this file\n+ * to you under the Apache License, Version 2.0 (the\n+ * \"License\"); you may not use this file except in compliance\n+ * with the License.  You may obtain a copy of the License at\n+ *\n+ *     http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+package org.apache.beam.runners.apex;\n+\n+import static com.google.common.base.Preconditions.checkArgument;\n+\n+import com.datatorrent.api.Attribute;\n+import com.datatorrent.api.Attribute.AttributeMap;\n+import com.datatorrent.api.DAG;\n+import com.datatorrent.api.StreamingApplication;\n+import com.google.common.annotations.VisibleForTesting;\n+import com.google.common.collect.Sets;\n+\n+import java.io.BufferedReader;\n+import java.io.ByteArrayOutputStream;\n+import java.io.File;\n+import java.io.FileInputStream;\n+import java.io.FileOutputStream;\n+import java.io.IOException;\n+import java.io.InputStream;\n+import java.io.InputStreamReader;\n+import java.io.OutputStream;\n+import java.io.Serializable;\n+import java.lang.reflect.AccessibleObject;\n+import java.lang.reflect.Field;\n+import java.net.URI;\n+import java.net.URL;\n+import java.net.URLClassLoader;\n+import java.nio.file.FileSystem;\n+import java.nio.file.FileSystems;\n+import java.nio.file.FileVisitResult;\n+import java.nio.file.Files;\n+import java.nio.file.Path;\n+import java.nio.file.attribute.BasicFileAttributes;\n+import java.util.ArrayList;\n+import java.util.Collections;\n+import java.util.HashMap;\n+import java.util.List;\n+import java.util.Map;\n+import java.util.Set;\n+import java.util.jar.JarFile;\n+\n+import org.apache.apex.api.EmbeddedAppLauncher;\n+import org.apache.apex.api.Launcher;\n+import org.apache.apex.api.Launcher.AppHandle;\n+import org.apache.apex.api.Launcher.LaunchMode;\n+import org.apache.apex.api.Launcher.LauncherException;\n+import org.apache.apex.api.Launcher.ShutdownMode;\n+import org.apache.apex.api.YarnAppLauncher;\n+import org.apache.commons.io.FileUtils;\n+import org.apache.commons.io.IOUtils;\n+import org.apache.commons.lang3.SerializationUtils;\n+import org.apache.hadoop.conf.Configuration;\n+import org.slf4j.Logger;\n+import org.slf4j.LoggerFactory;\n+\n+/**\n+ * Proxy to launch the YARN application through the hadoop script to run in the\n+ * pre-configured environment (class path, configuration, native libraries etc.).\n+ *\n+ * <p>The proxy takes the DAG and communicates with the Hadoop services to launch\n+ * it on the cluster.\n+ */\n+public class ApexYarnLauncher {\n+  private static final Logger LOG = LoggerFactory.getLogger(ApexYarnLauncher.class);\n+\n+  public AppHandle launchApp(StreamingApplication app) throws IOException {\n+\n+    List<File> jarsToShip = getYarnDeployDependencies();\n+    StringBuilder classpath = new StringBuilder();\n+    for (File path : jarsToShip) {\n+      if (path.isDirectory()) {\n+        File tmpJar = File.createTempFile(\"beam-runners-apex-\", \".jar\");\n+        createJar(path, tmpJar);\n+        tmpJar.deleteOnExit();\n+        path = tmpJar;\n+      }\n+      if (classpath.length() != 0) {\n+        classpath.append(':');\n+      }\n+      classpath.append(path.getAbsolutePath());\n+    }\n+\n+    EmbeddedAppLauncher<?> embeddedLauncher = Launcher.getLauncher(LaunchMode.EMBEDDED);\n+    DAG dag = embeddedLauncher.getDAG();","path":"runners/apex/src/main/java/org/apache/beam/runners/apex/ApexYarnLauncher.java","position":100,"original_position":100,"commit_id":"3595eabb6b68e11b7d59d61d399061f70689304c","original_commit_id":"3595eabb6b68e11b7d59d61d399061f70689304c","user":{"login":"tweise","id":263695,"avatar_url":"https://avatars.githubusercontent.com/u/263695?v=3","gravatar_id":"","url":"https://api.github.com/users/tweise","html_url":"https://github.com/tweise","followers_url":"https://api.github.com/users/tweise/followers","following_url":"https://api.github.com/users/tweise/following{/other_user}","gists_url":"https://api.github.com/users/tweise/gists{/gist_id}","starred_url":"https://api.github.com/users/tweise/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tweise/subscriptions","organizations_url":"https://api.github.com/users/tweise/orgs","repos_url":"https://api.github.com/users/tweise/repos","events_url":"https://api.github.com/users/tweise/events{/privacy}","received_events_url":"https://api.github.com/users/tweise/received_events","type":"User","site_admin":false},"body":"It is getDAG(), not createDAG(). I created a JIRA to look into this further: https://issues.apache.org/jira/browse/APEXCORE-587","created_at":"2016-12-07T18:35:05Z","updated_at":"2016-12-07T18:35:05Z","html_url":"https://github.com/apache/incubator-beam/pull/1517#discussion_r91356811","pull_request_url":"https://api.github.com/repos/apache/incubator-beam/pulls/1517","_links":{"self":{"href":"https://api.github.com/repos/apache/incubator-beam/pulls/comments/91356811"},"html":{"href":"https://github.com/apache/incubator-beam/pull/1517#discussion_r91356811"},"pull_request":{"href":"https://api.github.com/repos/apache/incubator-beam/pulls/1517"}}},"pull_request":{"url":"https://api.github.com/repos/apache/incubator-beam/pulls/1517","id":96736187,"html_url":"https://github.com/apache/incubator-beam/pull/1517","diff_url":"https://github.com/apache/incubator-beam/pull/1517.diff","patch_url":"https://github.com/apache/incubator-beam/pull/1517.patch","issue_url":"https://api.github.com/repos/apache/incubator-beam/issues/1517","number":1517,"state":"open","locked":false,"title":"[BEAM-830] ApexRunner launch on YARN cluster.","user":{"login":"tweise","id":263695,"avatar_url":"https://avatars.githubusercontent.com/u/263695?v=3","gravatar_id":"","url":"https://api.github.com/users/tweise","html_url":"https://github.com/tweise","followers_url":"https://api.github.com/users/tweise/followers","following_url":"https://api.github.com/users/tweise/following{/other_user}","gists_url":"https://api.github.com/users/tweise/gists{/gist_id}","starred_url":"https://api.github.com/users/tweise/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tweise/subscriptions","organizations_url":"https://api.github.com/users/tweise/orgs","repos_url":"https://api.github.com/users/tweise/repos","events_url":"https://api.github.com/users/tweise/events{/privacy}","received_events_url":"https://api.github.com/users/tweise/received_events","type":"User","site_admin":false},"body":"Be sure to do all of the following to help us incorporate your contribution\r\nquickly and easily:\r\n\r\n - [ ] Make sure the PR title is formatted like:\r\n   `[BEAM-<Jira issue #>] Description of pull request`\r\n - [ ] Make sure tests pass via `mvn clean verify`. (Even better, enable\r\n       Travis-CI on your fork and ensure the whole test matrix passes).\r\n - [ ] Replace `<Jira issue #>` in the title with the actual Jira issue\r\n       number, if there is one.\r\n - [ ] If this contribution is large, please file an Apache\r\n       [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.txt).\r\n\r\n---\r\nR: @kennknowles @dhalperi \r\n\r\nThis PR provides the support to run a Beam pipeline through the main method on the YARN cluster. Example:\r\n\r\n```\r\n$ mvn compile exec:java -Dexec.mainClass=org.apache.beam.examples.WordCount -Dexec.args=\"--inputFile=pom.xml --output=counts --runner=ApexRunner --embeddedExecution=false\" -Papex-runner\r\n```\r\n\r\nTo make that happen it has to do some magic with the class path to determine the dependencies that should be used with the Hadoop client. \r\n\r\nWould like to get these changes into the upcoming release.\r\n","created_at":"2016-12-06T14:42:39Z","updated_at":"2016-12-07T18:35:05Z","closed_at":null,"merged_at":null,"merge_commit_sha":"bcc8a3b5676919d3526f3b8f6f1d1052ba02e99f","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/incubator-beam/pulls/1517/commits","review_comments_url":"https://api.github.com/repos/apache/incubator-beam/pulls/1517/comments","review_comment_url":"https://api.github.com/repos/apache/incubator-beam/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/incubator-beam/issues/1517/comments","statuses_url":"https://api.github.com/repos/apache/incubator-beam/statuses/3595eabb6b68e11b7d59d61d399061f70689304c","head":{"label":"tweise:BEAM-830","ref":"BEAM-830","sha":"3595eabb6b68e11b7d59d61d399061f70689304c","user":{"login":"tweise","id":263695,"avatar_url":"https://avatars.githubusercontent.com/u/263695?v=3","gravatar_id":"","url":"https://api.github.com/users/tweise","html_url":"https://github.com/tweise","followers_url":"https://api.github.com/users/tweise/followers","following_url":"https://api.github.com/users/tweise/following{/other_user}","gists_url":"https://api.github.com/users/tweise/gists{/gist_id}","starred_url":"https://api.github.com/users/tweise/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tweise/subscriptions","organizations_url":"https://api.github.com/users/tweise/orgs","repos_url":"https://api.github.com/users/tweise/repos","events_url":"https://api.github.com/users/tweise/events{/privacy}","received_events_url":"https://api.github.com/users/tweise/received_events","type":"User","site_admin":false},"repo":{"id":59848217,"name":"incubator-beam","full_name":"tweise/incubator-beam","owner":{"login":"tweise","id":263695,"avatar_url":"https://avatars.githubusercontent.com/u/263695?v=3","gravatar_id":"","url":"https://api.github.com/users/tweise","html_url":"https://github.com/tweise","followers_url":"https://api.github.com/users/tweise/followers","following_url":"https://api.github.com/users/tweise/following{/other_user}","gists_url":"https://api.github.com/users/tweise/gists{/gist_id}","starred_url":"https://api.github.com/users/tweise/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tweise/subscriptions","organizations_url":"https://api.github.com/users/tweise/orgs","repos_url":"https://api.github.com/users/tweise/repos","events_url":"https://api.github.com/users/tweise/events{/privacy}","received_events_url":"https://api.github.com/users/tweise/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/tweise/incubator-beam","description":"Mirror of Apache Beam (Incubating)","fork":true,"url":"https://api.github.com/repos/tweise/incubator-beam","forks_url":"https://api.github.com/repos/tweise/incubator-beam/forks","keys_url":"https://api.github.com/repos/tweise/incubator-beam/keys{/key_id}","collaborators_url":"https://api.github.com/repos/tweise/incubator-beam/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/tweise/incubator-beam/teams","hooks_url":"https://api.github.com/repos/tweise/incubator-beam/hooks","issue_events_url":"https://api.github.com/repos/tweise/incubator-beam/issues/events{/number}","events_url":"https://api.github.com/repos/tweise/incubator-beam/events","assignees_url":"https://api.github.com/repos/tweise/incubator-beam/assignees{/user}","branches_url":"https://api.github.com/repos/tweise/incubator-beam/branches{/branch}","tags_url":"https://api.github.com/repos/tweise/incubator-beam/tags","blobs_url":"https://api.github.com/repos/tweise/incubator-beam/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/tweise/incubator-beam/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/tweise/incubator-beam/git/refs{/sha}","trees_url":"https://api.github.com/repos/tweise/incubator-beam/git/trees{/sha}","statuses_url":"https://api.github.com/repos/tweise/incubator-beam/statuses/{sha}","languages_url":"https://api.github.com/repos/tweise/incubator-beam/languages","stargazers_url":"https://api.github.com/repos/tweise/incubator-beam/stargazers","contributors_url":"https://api.github.com/repos/tweise/incubator-beam/contributors","subscribers_url":"https://api.github.com/repos/tweise/incubator-beam/subscribers","subscription_url":"https://api.github.com/repos/tweise/incubator-beam/subscription","commits_url":"https://api.github.com/repos/tweise/incubator-beam/commits{/sha}","git_commits_url":"https://api.github.com/repos/tweise/incubator-beam/git/commits{/sha}","comments_url":"https://api.github.com/repos/tweise/incubator-beam/comments{/number}","issue_comment_url":"https://api.github.com/repos/tweise/incubator-beam/issues/comments{/number}","contents_url":"https://api.github.com/repos/tweise/incubator-beam/contents/{+path}","compare_url":"https://api.github.com/repos/tweise/incubator-beam/compare/{base}...{head}","merges_url":"https://api.github.com/repos/tweise/incubator-beam/merges","archive_url":"https://api.github.com/repos/tweise/incubator-beam/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/tweise/incubator-beam/downloads","issues_url":"https://api.github.com/repos/tweise/incubator-beam/issues{/number}","pulls_url":"https://api.github.com/repos/tweise/incubator-beam/pulls{/number}","milestones_url":"https://api.github.com/repos/tweise/incubator-beam/milestones{/number}","notifications_url":"https://api.github.com/repos/tweise/incubator-beam/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/tweise/incubator-beam/labels{/name}","releases_url":"https://api.github.com/repos/tweise/incubator-beam/releases{/id}","deployments_url":"https://api.github.com/repos/tweise/incubator-beam/deployments","created_at":"2016-05-27T16:24:56Z","updated_at":"2016-05-27T16:25:01Z","pushed_at":"2016-12-07T01:14:06Z","git_url":"git://github.com/tweise/incubator-beam.git","ssh_url":"git@github.com:tweise/incubator-beam.git","clone_url":"https://github.com/tweise/incubator-beam.git","svn_url":"https://github.com/tweise/incubator-beam","homepage":null,"size":22394,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"afedd68e806830549724dfc0f2565d756cc6b46d","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":50904245,"name":"incubator-beam","full_name":"apache/incubator-beam","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/incubator-beam","description":"Mirror of Apache Beam (Incubating)","fork":false,"url":"https://api.github.com/repos/apache/incubator-beam","forks_url":"https://api.github.com/repos/apache/incubator-beam/forks","keys_url":"https://api.github.com/repos/apache/incubator-beam/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/incubator-beam/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/incubator-beam/teams","hooks_url":"https://api.github.com/repos/apache/incubator-beam/hooks","issue_events_url":"https://api.github.com/repos/apache/incubator-beam/issues/events{/number}","events_url":"https://api.github.com/repos/apache/incubator-beam/events","assignees_url":"https://api.github.com/repos/apache/incubator-beam/assignees{/user}","branches_url":"https://api.github.com/repos/apache/incubator-beam/branches{/branch}","tags_url":"https://api.github.com/repos/apache/incubator-beam/tags","blobs_url":"https://api.github.com/repos/apache/incubator-beam/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/incubator-beam/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/incubator-beam/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/incubator-beam/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/incubator-beam/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/incubator-beam/languages","stargazers_url":"https://api.github.com/repos/apache/incubator-beam/stargazers","contributors_url":"https://api.github.com/repos/apache/incubator-beam/contributors","subscribers_url":"https://api.github.com/repos/apache/incubator-beam/subscribers","subscription_url":"https://api.github.com/repos/apache/incubator-beam/subscription","commits_url":"https://api.github.com/repos/apache/incubator-beam/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/incubator-beam/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/incubator-beam/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/incubator-beam/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/incubator-beam/contents/{+path}","compare_url":"https://api.github.com/repos/apache/incubator-beam/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/incubator-beam/merges","archive_url":"https://api.github.com/repos/apache/incubator-beam/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/incubator-beam/downloads","issues_url":"https://api.github.com/repos/apache/incubator-beam/issues{/number}","pulls_url":"https://api.github.com/repos/apache/incubator-beam/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/incubator-beam/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/incubator-beam/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/incubator-beam/labels{/name}","releases_url":"https://api.github.com/repos/apache/incubator-beam/releases{/id}","deployments_url":"https://api.github.com/repos/apache/incubator-beam/deployments","created_at":"2016-02-02T08:00:06Z","updated_at":"2016-12-07T13:59:18Z","pushed_at":"2016-12-07T18:15:42Z","git_url":"git://github.com/apache/incubator-beam.git","ssh_url":"git@github.com:apache/incubator-beam.git","clone_url":"https://github.com/apache/incubator-beam.git","svn_url":"https://github.com/apache/incubator-beam","homepage":null,"size":37403,"stargazers_count":518,"watchers_count":518,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":306,"mirror_url":"git://git.apache.org/incubator-beam.git","open_issues_count":57,"forks":306,"open_issues":57,"watchers":518,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/incubator-beam/pulls/1517"},"html":{"href":"https://github.com/apache/incubator-beam/pull/1517"},"issue":{"href":"https://api.github.com/repos/apache/incubator-beam/issues/1517"},"comments":{"href":"https://api.github.com/repos/apache/incubator-beam/issues/1517/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/incubator-beam/pulls/1517/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/incubator-beam/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/incubator-beam/pulls/1517/commits"},"statuses":{"href":"https://api.github.com/repos/apache/incubator-beam/statuses/3595eabb6b68e11b7d59d61d399061f70689304c"}}}}
{"actor":{"display_login":"tweise"}}