{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/186052495","pull_request_review_id":117567591,"id":186052495,"diff_hunk":"@@ -378,6 +408,135 @@ case class ArrayContains(left: Expression, right: Expression)\n   override def prettyName: String = \"array_contains\"\n }\n \n+/**\n+ * Checks if the two arrays contain at least one common element.\n+ */\n+// scalastyle:off line.size.limit\n+@ExpressionDescription(\n+  usage = \"_FUNC_(a1, a2) - Returns true if a1 contains at least an element present also in a2. If the arrays have no common element and either of them contains a null element null is returned, false otherwise.\",\n+  examples = \"\"\"\n+    Examples:\n+      > SELECT _FUNC_(array(1, 2, 3), array(3, 4, 5));\n+       true\n+  \"\"\", since = \"2.4.0\")\n+// scalastyle:off line.size.limit\n+case class ArraysOverlap(left: Expression, right: Expression)\n+  extends BinaryArrayExpressionWithImplicitCast {\n+\n+  override def dataType: DataType = BooleanType\n+\n+  override def nullable: Boolean = {\n+    left.nullable || right.nullable || left.dataType.asInstanceOf[ArrayType].containsNull ||\n+      right.dataType.asInstanceOf[ArrayType].containsNull\n+  }\n+\n+  override def nullSafeEval(a1: Any, a2: Any): Any = {\n+    var hasNull = false\n+    val arr1 = a1.asInstanceOf[ArrayData]\n+    val arr2 = a2.asInstanceOf[ArrayData]\n+    if (arr1.numElements() > 0) {\n+      val set2 = new mutable.HashSet[Any]\n+      arr2.foreach(elementType, (_, v) =>\n+        if (v == null) {\n+          hasNull = true\n+        } else {\n+          set2 += v\n+        })\n+      arr1.foreach(elementType, (_, v1) =>\n+        if (v1 == null) {\n+          hasNull = true\n+        } else if (set2.contains(v1)) {\n+          return true\n+        }\n+      )\n+    } else if (containsNull(arr2, right.dataType.asInstanceOf[ArrayType])) {\n+      hasNull = true\n+    }\n+    if (hasNull) {\n+      null\n+    } else {\n+      false\n+    }\n+  }\n+\n+  def containsNull(arr: ArrayData, dt: ArrayType): Boolean = {\n+    if (dt.containsNull) {\n+      arr.foreach(elementType, (_, v) =>\n+        if (v == null) {\n+          return true\n+        }\n+      )\n+    }\n+    false\n+  }\n+\n+  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {\n+    nullSafeCodeGen(ctx, ev, (a1, a2) => {\n+      val i1 = ctx.freshName(\"i\")\n+      val i2 = ctx.freshName(\"i\")\n+      val getValue1 = CodeGenerator.getValue(a1, elementType, i1)\n+      val getValue2 = CodeGenerator.getValue(a2, elementType, i2)\n+      val leftEmptyCode = if (right.dataType.asInstanceOf[ArrayType].containsNull) {\n+        s\"\"\"\n+           |else {\n+           |  for (int $i2 = 0; $i2 < $a2.numElements(); $i2 ++) {\n+           |    if ($a2.isNullAt($i2)) {\n+           |      ${ev.isNull} = true;\n+           |      break;\n+           |    }\n+           |  }\n+           |}\n+         \"\"\".stripMargin\n+      } else {\n+        \"\"\n+      }\n+      val javaElementClass = CodeGenerator.boxedType(elementType)\n+      val javaSet = classOf[java.util.HashSet[_]].getName\n+      val set2 = ctx.freshName(\"set\")\n+      s\"\"\"\n+         |if ($a1.numElements() > 0) {\n+         |  $javaSet<$javaElementClass> $set2 = new $javaSet<$javaElementClass>();\n+         |  for (int $i2 = 0; $i2 < $a2.numElements(); $i2 ++) {\n+         |    ${nullSafeElementCodegen(right.dataType.asInstanceOf[ArrayType], a2, i2,\n+                  s\"$set2.add($getValue2);\", s\"${ev.isNull} = true;\")}\n+         |  }\n+         |  for (int $i1 = 0; $i1 < $a1.numElements(); $i1 ++) {\n+         |    ${nullSafeElementCodegen(left.dataType.asInstanceOf[ArrayType], a1, i1,","path":"sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala","position":144,"original_position":144,"commit_id":"4a18ba89c4489de5efe34eb31108665474f70f23","original_commit_id":"4a18ba89c4489de5efe34eb31108665474f70f23","user":{"login":"cloud-fan","id":3182036,"avatar_url":"https://avatars3.githubusercontent.com/u/3182036?v=4","gravatar_id":"","url":"https://api.github.com/users/cloud-fan","html_url":"https://github.com/cloud-fan","followers_url":"https://api.github.com/users/cloud-fan/followers","following_url":"https://api.github.com/users/cloud-fan/following{/other_user}","gists_url":"https://api.github.com/users/cloud-fan/gists{/gist_id}","starred_url":"https://api.github.com/users/cloud-fan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cloud-fan/subscriptions","organizations_url":"https://api.github.com/users/cloud-fan/orgs","repos_url":"https://api.github.com/users/cloud-fan/repos","events_url":"https://api.github.com/users/cloud-fan/events{/privacy}","received_events_url":"https://api.github.com/users/cloud-fan/received_events","type":"User","site_admin":false},"body":"ditto","created_at":"2018-05-04T11:12:21Z","updated_at":"2018-05-04T11:12:21Z","html_url":"https://github.com/apache/spark/pull/21028#discussion_r186052495","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/21028","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/186052495"},"html":{"href":"https://github.com/apache/spark/pull/21028#discussion_r186052495"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/21028"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/21028","id":180634191,"html_url":"https://github.com/apache/spark/pull/21028","diff_url":"https://github.com/apache/spark/pull/21028.diff","patch_url":"https://github.com/apache/spark/pull/21028.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/21028","number":21028,"state":"open","locked":false,"title":"[SPARK-23922][SQL] Add arrays_overlap function","user":{"login":"mgaido91","id":8821783,"avatar_url":"https://avatars1.githubusercontent.com/u/8821783?v=4","gravatar_id":"","url":"https://api.github.com/users/mgaido91","html_url":"https://github.com/mgaido91","followers_url":"https://api.github.com/users/mgaido91/followers","following_url":"https://api.github.com/users/mgaido91/following{/other_user}","gists_url":"https://api.github.com/users/mgaido91/gists{/gist_id}","starred_url":"https://api.github.com/users/mgaido91/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mgaido91/subscriptions","organizations_url":"https://api.github.com/users/mgaido91/orgs","repos_url":"https://api.github.com/users/mgaido91/repos","events_url":"https://api.github.com/users/mgaido91/events{/privacy}","received_events_url":"https://api.github.com/users/mgaido91/received_events","type":"User","site_admin":false},"body":"## What changes were proposed in this pull request?\r\n\r\nThe PR adds the function `arrays_overlap`. This function returns `true` if the input arrays contain a non-null common element; if not, it returns `null` if any of the arrays contains a `null` element, `false` otherwise.\r\n\r\n## How was this patch tested?\r\n\r\nadded UTs","created_at":"2018-04-10T15:06:21Z","updated_at":"2018-05-04T11:12:21Z","closed_at":null,"merged_at":null,"merge_commit_sha":"f5a709a3fe4bc4a7a4d66bdd11df2f19bfb28d8e","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/21028/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/21028/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/21028/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/4a18ba89c4489de5efe34eb31108665474f70f23","head":{"label":"mgaido91:SPARK-23922","ref":"SPARK-23922","sha":"4a18ba89c4489de5efe34eb31108665474f70f23","user":{"login":"mgaido91","id":8821783,"avatar_url":"https://avatars1.githubusercontent.com/u/8821783?v=4","gravatar_id":"","url":"https://api.github.com/users/mgaido91","html_url":"https://github.com/mgaido91","followers_url":"https://api.github.com/users/mgaido91/followers","following_url":"https://api.github.com/users/mgaido91/following{/other_user}","gists_url":"https://api.github.com/users/mgaido91/gists{/gist_id}","starred_url":"https://api.github.com/users/mgaido91/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mgaido91/subscriptions","organizations_url":"https://api.github.com/users/mgaido91/orgs","repos_url":"https://api.github.com/users/mgaido91/repos","events_url":"https://api.github.com/users/mgaido91/events{/privacy}","received_events_url":"https://api.github.com/users/mgaido91/received_events","type":"User","site_admin":false},"repo":{"id":94518128,"name":"spark","full_name":"mgaido91/spark","owner":{"login":"mgaido91","id":8821783,"avatar_url":"https://avatars1.githubusercontent.com/u/8821783?v=4","gravatar_id":"","url":"https://api.github.com/users/mgaido91","html_url":"https://github.com/mgaido91","followers_url":"https://api.github.com/users/mgaido91/followers","following_url":"https://api.github.com/users/mgaido91/following{/other_user}","gists_url":"https://api.github.com/users/mgaido91/gists{/gist_id}","starred_url":"https://api.github.com/users/mgaido91/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mgaido91/subscriptions","organizations_url":"https://api.github.com/users/mgaido91/orgs","repos_url":"https://api.github.com/users/mgaido91/repos","events_url":"https://api.github.com/users/mgaido91/events{/privacy}","received_events_url":"https://api.github.com/users/mgaido91/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/mgaido91/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/mgaido91/spark","forks_url":"https://api.github.com/repos/mgaido91/spark/forks","keys_url":"https://api.github.com/repos/mgaido91/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mgaido91/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mgaido91/spark/teams","hooks_url":"https://api.github.com/repos/mgaido91/spark/hooks","issue_events_url":"https://api.github.com/repos/mgaido91/spark/issues/events{/number}","events_url":"https://api.github.com/repos/mgaido91/spark/events","assignees_url":"https://api.github.com/repos/mgaido91/spark/assignees{/user}","branches_url":"https://api.github.com/repos/mgaido91/spark/branches{/branch}","tags_url":"https://api.github.com/repos/mgaido91/spark/tags","blobs_url":"https://api.github.com/repos/mgaido91/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mgaido91/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mgaido91/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/mgaido91/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mgaido91/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/mgaido91/spark/languages","stargazers_url":"https://api.github.com/repos/mgaido91/spark/stargazers","contributors_url":"https://api.github.com/repos/mgaido91/spark/contributors","subscribers_url":"https://api.github.com/repos/mgaido91/spark/subscribers","subscription_url":"https://api.github.com/repos/mgaido91/spark/subscription","commits_url":"https://api.github.com/repos/mgaido91/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/mgaido91/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/mgaido91/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/mgaido91/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/mgaido91/spark/contents/{+path}","compare_url":"https://api.github.com/repos/mgaido91/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mgaido91/spark/merges","archive_url":"https://api.github.com/repos/mgaido91/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mgaido91/spark/downloads","issues_url":"https://api.github.com/repos/mgaido91/spark/issues{/number}","pulls_url":"https://api.github.com/repos/mgaido91/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/mgaido91/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/mgaido91/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mgaido91/spark/labels{/name}","releases_url":"https://api.github.com/repos/mgaido91/spark/releases{/id}","deployments_url":"https://api.github.com/repos/mgaido91/spark/deployments","created_at":"2017-06-16T07:34:38Z","updated_at":"2017-06-16T07:35:05Z","pushed_at":"2018-05-04T09:15:49Z","git_url":"git://github.com/mgaido91/spark.git","ssh_url":"git@github.com:mgaido91/spark.git","clone_url":"https://github.com/mgaido91/spark.git","svn_url":"https://github.com/mgaido91/spark","homepage":null,"size":267515,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"109935fc5d8b3d381bb1b09a4a570040a0a1846f","user":{"login":"apache","id":47359,"avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","deployments_url":"https://api.github.com/repos/apache/spark/deployments","created_at":"2014-02-25T08:00:08Z","updated_at":"2018-05-04T11:08:34Z","pushed_at":"2018-05-04T09:59:52Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":269446,"stargazers_count":17159,"watchers_count":17159,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":15718,"mirror_url":null,"archived":false,"open_issues_count":539,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":15718,"open_issues":539,"watchers":17159,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/21028"},"html":{"href":"https://github.com/apache/spark/pull/21028"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/21028"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/21028/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/21028/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/21028/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/4a18ba89c4489de5efe34eb31108665474f70f23"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"cloud-fan"}}