{"action":"created","comment":{"url":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/comments/73025685","id":73025685,"diff_hunk":"@@ -348,9 +414,51 @@ private static void handleStandardArgument(Props jobProps,\n       }\n     }\n \n+    info(\"Final classpath: \" + classPath);\n     return classPath;\n   }\n \n+  private String[] getSparkHomeConf() {\n+    String sparkHome = null;\n+    String sparkConf = null;\n+    // If user has specified version in job property. e.g. spark.version=1.6.0\n+    String jobSparkVer = getJobProps().get(SparkJobArg.SPARK_VERSION.azPropName);\n+    if (jobSparkVer != null) {\n+      // Spark jobtype supports this version through plugin's jobtype config\n+      // e.g. spark.1.6.0.home=/path_to_spark/ in commonprivate.properties\n+      sparkHome = getSysProps().get(\"spark.\" + jobSparkVer + \".home\");\n+      if (sparkHome != null) {\n+        sparkConf = getSysProps().get(\"spark.\" + jobSparkVer + \".conf\");\n+        if (sparkConf == null) {\n+          sparkConf = sparkHome + \"/conf\";\n+        }\n+        info(\"Using job specific spark: \" + sparkHome + \" and conf: \" + sparkConf);\n+        // Override the SPARK_HOME SPARK_CONF_DIR env for HadoopSecureSparkWrapper process(spark client)\n+        getJobProps().put(\"env.\" + SPARK_HOME_ENV_VAR, sparkHome);","path":"plugins/jobtype/src/azkaban/jobtype/HadoopSparkJob.java","position":173,"original_position":173,"commit_id":"2c700266a1ea2db43e69eb5007746757a21f8f1a","original_commit_id":"2c700266a1ea2db43e69eb5007746757a21f8f1a","user":{"login":"Victsm","id":4633527,"avatar_url":"https://avatars.githubusercontent.com/u/4633527?v=3","gravatar_id":"","url":"https://api.github.com/users/Victsm","html_url":"https://github.com/Victsm","followers_url":"https://api.github.com/users/Victsm/followers","following_url":"https://api.github.com/users/Victsm/following{/other_user}","gists_url":"https://api.github.com/users/Victsm/gists{/gist_id}","starred_url":"https://api.github.com/users/Victsm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Victsm/subscriptions","organizations_url":"https://api.github.com/users/Victsm/orgs","repos_url":"https://api.github.com/users/Victsm/repos","events_url":"https://api.github.com/users/Victsm/events{/privacy}","received_events_url":"https://api.github.com/users/Victsm/received_events","type":"User","site_admin":false},"body":"Even though we configure spark home dir and spark conf dir in plugin properties, it would be better to add a check here for the existence of these directories.","created_at":"2016-08-01T18:15:48Z","updated_at":"2016-08-01T18:15:48Z","html_url":"https://github.com/azkaban/azkaban-plugins/pull/231#discussion_r73025685","pull_request_url":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/231","_links":{"self":{"href":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/comments/73025685"},"html":{"href":"https://github.com/azkaban/azkaban-plugins/pull/231#discussion_r73025685"},"pull_request":{"href":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/231"}}},"pull_request":{"url":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/231","id":79245456,"html_url":"https://github.com/azkaban/azkaban-plugins/pull/231","diff_url":"https://github.com/azkaban/azkaban-plugins/pull/231.diff","patch_url":"https://github.com/azkaban/azkaban-plugins/pull/231.patch","issue_url":"https://api.github.com/repos/azkaban/azkaban-plugins/issues/231","number":231,"state":"open","locked":false,"title":"update spark jobtype plugin to support multiple spark version","user":{"login":"fangshil","id":2416746,"avatar_url":"https://avatars.githubusercontent.com/u/2416746?v=3","gravatar_id":"","url":"https://api.github.com/users/fangshil","html_url":"https://github.com/fangshil","followers_url":"https://api.github.com/users/fangshil/followers","following_url":"https://api.github.com/users/fangshil/following{/other_user}","gists_url":"https://api.github.com/users/fangshil/gists{/gist_id}","starred_url":"https://api.github.com/users/fangshil/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fangshil/subscriptions","organizations_url":"https://api.github.com/users/fangshil/orgs","repos_url":"https://api.github.com/users/fangshil/repos","events_url":"https://api.github.com/users/fangshil/events{/privacy}","received_events_url":"https://api.github.com/users/fangshil/received_events","type":"User","site_admin":false},"body":"","created_at":"2016-07-28T18:32:25Z","updated_at":"2016-08-01T18:15:48Z","closed_at":null,"merged_at":null,"merge_commit_sha":"11d2a9ef1a99f7c210e83460415abc812c910d87","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/231/commits","review_comments_url":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/231/comments","review_comment_url":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/comments{/number}","comments_url":"https://api.github.com/repos/azkaban/azkaban-plugins/issues/231/comments","statuses_url":"https://api.github.com/repos/azkaban/azkaban-plugins/statuses/2c700266a1ea2db43e69eb5007746757a21f8f1a","head":{"label":"fangshil:master","ref":"master","sha":"2c700266a1ea2db43e69eb5007746757a21f8f1a","user":{"login":"fangshil","id":2416746,"avatar_url":"https://avatars.githubusercontent.com/u/2416746?v=3","gravatar_id":"","url":"https://api.github.com/users/fangshil","html_url":"https://github.com/fangshil","followers_url":"https://api.github.com/users/fangshil/followers","following_url":"https://api.github.com/users/fangshil/following{/other_user}","gists_url":"https://api.github.com/users/fangshil/gists{/gist_id}","starred_url":"https://api.github.com/users/fangshil/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fangshil/subscriptions","organizations_url":"https://api.github.com/users/fangshil/orgs","repos_url":"https://api.github.com/users/fangshil/repos","events_url":"https://api.github.com/users/fangshil/events{/privacy}","received_events_url":"https://api.github.com/users/fangshil/received_events","type":"User","site_admin":false},"repo":{"id":64418597,"name":"azkaban-plugins","full_name":"fangshil/azkaban-plugins","owner":{"login":"fangshil","id":2416746,"avatar_url":"https://avatars.githubusercontent.com/u/2416746?v=3","gravatar_id":"","url":"https://api.github.com/users/fangshil","html_url":"https://github.com/fangshil","followers_url":"https://api.github.com/users/fangshil/followers","following_url":"https://api.github.com/users/fangshil/following{/other_user}","gists_url":"https://api.github.com/users/fangshil/gists{/gist_id}","starred_url":"https://api.github.com/users/fangshil/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fangshil/subscriptions","organizations_url":"https://api.github.com/users/fangshil/orgs","repos_url":"https://api.github.com/users/fangshil/repos","events_url":"https://api.github.com/users/fangshil/events{/privacy}","received_events_url":"https://api.github.com/users/fangshil/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/fangshil/azkaban-plugins","description":"Plugins for Azkaban.","fork":true,"url":"https://api.github.com/repos/fangshil/azkaban-plugins","forks_url":"https://api.github.com/repos/fangshil/azkaban-plugins/forks","keys_url":"https://api.github.com/repos/fangshil/azkaban-plugins/keys{/key_id}","collaborators_url":"https://api.github.com/repos/fangshil/azkaban-plugins/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/fangshil/azkaban-plugins/teams","hooks_url":"https://api.github.com/repos/fangshil/azkaban-plugins/hooks","issue_events_url":"https://api.github.com/repos/fangshil/azkaban-plugins/issues/events{/number}","events_url":"https://api.github.com/repos/fangshil/azkaban-plugins/events","assignees_url":"https://api.github.com/repos/fangshil/azkaban-plugins/assignees{/user}","branches_url":"https://api.github.com/repos/fangshil/azkaban-plugins/branches{/branch}","tags_url":"https://api.github.com/repos/fangshil/azkaban-plugins/tags","blobs_url":"https://api.github.com/repos/fangshil/azkaban-plugins/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/fangshil/azkaban-plugins/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/fangshil/azkaban-plugins/git/refs{/sha}","trees_url":"https://api.github.com/repos/fangshil/azkaban-plugins/git/trees{/sha}","statuses_url":"https://api.github.com/repos/fangshil/azkaban-plugins/statuses/{sha}","languages_url":"https://api.github.com/repos/fangshil/azkaban-plugins/languages","stargazers_url":"https://api.github.com/repos/fangshil/azkaban-plugins/stargazers","contributors_url":"https://api.github.com/repos/fangshil/azkaban-plugins/contributors","subscribers_url":"https://api.github.com/repos/fangshil/azkaban-plugins/subscribers","subscription_url":"https://api.github.com/repos/fangshil/azkaban-plugins/subscription","commits_url":"https://api.github.com/repos/fangshil/azkaban-plugins/commits{/sha}","git_commits_url":"https://api.github.com/repos/fangshil/azkaban-plugins/git/commits{/sha}","comments_url":"https://api.github.com/repos/fangshil/azkaban-plugins/comments{/number}","issue_comment_url":"https://api.github.com/repos/fangshil/azkaban-plugins/issues/comments{/number}","contents_url":"https://api.github.com/repos/fangshil/azkaban-plugins/contents/{+path}","compare_url":"https://api.github.com/repos/fangshil/azkaban-plugins/compare/{base}...{head}","merges_url":"https://api.github.com/repos/fangshil/azkaban-plugins/merges","archive_url":"https://api.github.com/repos/fangshil/azkaban-plugins/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/fangshil/azkaban-plugins/downloads","issues_url":"https://api.github.com/repos/fangshil/azkaban-plugins/issues{/number}","pulls_url":"https://api.github.com/repos/fangshil/azkaban-plugins/pulls{/number}","milestones_url":"https://api.github.com/repos/fangshil/azkaban-plugins/milestones{/number}","notifications_url":"https://api.github.com/repos/fangshil/azkaban-plugins/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/fangshil/azkaban-plugins/labels{/name}","releases_url":"https://api.github.com/repos/fangshil/azkaban-plugins/releases{/id}","deployments_url":"https://api.github.com/repos/fangshil/azkaban-plugins/deployments","created_at":"2016-07-28T18:25:27Z","updated_at":"2016-07-28T18:25:31Z","pushed_at":"2016-07-30T10:01:11Z","git_url":"git://github.com/fangshil/azkaban-plugins.git","ssh_url":"git@github.com:fangshil/azkaban-plugins.git","clone_url":"https://github.com/fangshil/azkaban-plugins.git","svn_url":"https://github.com/fangshil/azkaban-plugins","homepage":"https://azkaban.github.io","size":306721,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"azkaban:master","ref":"master","sha":"7dff4babdddf1f99af3ccc6f19bd01d30a2290c4","user":{"login":"azkaban","id":213618,"avatar_url":"https://avatars.githubusercontent.com/u/213618?v=3","gravatar_id":"","url":"https://api.github.com/users/azkaban","html_url":"https://github.com/azkaban","followers_url":"https://api.github.com/users/azkaban/followers","following_url":"https://api.github.com/users/azkaban/following{/other_user}","gists_url":"https://api.github.com/users/azkaban/gists{/gist_id}","starred_url":"https://api.github.com/users/azkaban/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/azkaban/subscriptions","organizations_url":"https://api.github.com/users/azkaban/orgs","repos_url":"https://api.github.com/users/azkaban/repos","events_url":"https://api.github.com/users/azkaban/events{/privacy}","received_events_url":"https://api.github.com/users/azkaban/received_events","type":"Organization","site_admin":false},"repo":{"id":7566859,"name":"azkaban-plugins","full_name":"azkaban/azkaban-plugins","owner":{"login":"azkaban","id":213618,"avatar_url":"https://avatars.githubusercontent.com/u/213618?v=3","gravatar_id":"","url":"https://api.github.com/users/azkaban","html_url":"https://github.com/azkaban","followers_url":"https://api.github.com/users/azkaban/followers","following_url":"https://api.github.com/users/azkaban/following{/other_user}","gists_url":"https://api.github.com/users/azkaban/gists{/gist_id}","starred_url":"https://api.github.com/users/azkaban/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/azkaban/subscriptions","organizations_url":"https://api.github.com/users/azkaban/orgs","repos_url":"https://api.github.com/users/azkaban/repos","events_url":"https://api.github.com/users/azkaban/events{/privacy}","received_events_url":"https://api.github.com/users/azkaban/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/azkaban/azkaban-plugins","description":"Plugins for Azkaban.","fork":false,"url":"https://api.github.com/repos/azkaban/azkaban-plugins","forks_url":"https://api.github.com/repos/azkaban/azkaban-plugins/forks","keys_url":"https://api.github.com/repos/azkaban/azkaban-plugins/keys{/key_id}","collaborators_url":"https://api.github.com/repos/azkaban/azkaban-plugins/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/azkaban/azkaban-plugins/teams","hooks_url":"https://api.github.com/repos/azkaban/azkaban-plugins/hooks","issue_events_url":"https://api.github.com/repos/azkaban/azkaban-plugins/issues/events{/number}","events_url":"https://api.github.com/repos/azkaban/azkaban-plugins/events","assignees_url":"https://api.github.com/repos/azkaban/azkaban-plugins/assignees{/user}","branches_url":"https://api.github.com/repos/azkaban/azkaban-plugins/branches{/branch}","tags_url":"https://api.github.com/repos/azkaban/azkaban-plugins/tags","blobs_url":"https://api.github.com/repos/azkaban/azkaban-plugins/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/azkaban/azkaban-plugins/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/azkaban/azkaban-plugins/git/refs{/sha}","trees_url":"https://api.github.com/repos/azkaban/azkaban-plugins/git/trees{/sha}","statuses_url":"https://api.github.com/repos/azkaban/azkaban-plugins/statuses/{sha}","languages_url":"https://api.github.com/repos/azkaban/azkaban-plugins/languages","stargazers_url":"https://api.github.com/repos/azkaban/azkaban-plugins/stargazers","contributors_url":"https://api.github.com/repos/azkaban/azkaban-plugins/contributors","subscribers_url":"https://api.github.com/repos/azkaban/azkaban-plugins/subscribers","subscription_url":"https://api.github.com/repos/azkaban/azkaban-plugins/subscription","commits_url":"https://api.github.com/repos/azkaban/azkaban-plugins/commits{/sha}","git_commits_url":"https://api.github.com/repos/azkaban/azkaban-plugins/git/commits{/sha}","comments_url":"https://api.github.com/repos/azkaban/azkaban-plugins/comments{/number}","issue_comment_url":"https://api.github.com/repos/azkaban/azkaban-plugins/issues/comments{/number}","contents_url":"https://api.github.com/repos/azkaban/azkaban-plugins/contents/{+path}","compare_url":"https://api.github.com/repos/azkaban/azkaban-plugins/compare/{base}...{head}","merges_url":"https://api.github.com/repos/azkaban/azkaban-plugins/merges","archive_url":"https://api.github.com/repos/azkaban/azkaban-plugins/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/azkaban/azkaban-plugins/downloads","issues_url":"https://api.github.com/repos/azkaban/azkaban-plugins/issues{/number}","pulls_url":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls{/number}","milestones_url":"https://api.github.com/repos/azkaban/azkaban-plugins/milestones{/number}","notifications_url":"https://api.github.com/repos/azkaban/azkaban-plugins/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/azkaban/azkaban-plugins/labels{/name}","releases_url":"https://api.github.com/repos/azkaban/azkaban-plugins/releases{/id}","deployments_url":"https://api.github.com/repos/azkaban/azkaban-plugins/deployments","created_at":"2013-01-11T20:53:36Z","updated_at":"2016-06-15T12:57:47Z","pushed_at":"2016-07-30T10:01:11Z","git_url":"git://github.com/azkaban/azkaban-plugins.git","ssh_url":"git@github.com:azkaban/azkaban-plugins.git","clone_url":"https://github.com/azkaban/azkaban-plugins.git","svn_url":"https://github.com/azkaban/azkaban-plugins","homepage":"https://azkaban.github.io","size":366788,"stargazers_count":47,"watchers_count":47,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":91,"mirror_url":null,"open_issues_count":28,"forks":91,"open_issues":28,"watchers":47,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/231"},"html":{"href":"https://github.com/azkaban/azkaban-plugins/pull/231"},"issue":{"href":"https://api.github.com/repos/azkaban/azkaban-plugins/issues/231"},"comments":{"href":"https://api.github.com/repos/azkaban/azkaban-plugins/issues/231/comments"},"review_comments":{"href":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/231/comments"},"review_comment":{"href":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/231/commits"},"statuses":{"href":"https://api.github.com/repos/azkaban/azkaban-plugins/statuses/2c700266a1ea2db43e69eb5007746757a21f8f1a"}}}}
{"actor":{"display_login":"Victsm"}}