{"action":"created","comment":{"url":"https://api.github.com/repos/apache/parquet-mr/pulls/comments/69021725","id":69021725,"diff_hunk":"@@ -0,0 +1,116 @@\n+/*\n+ *  Licensed to the Apache Software Foundation (ASF) under one\n+ *  or more contributor license agreements.  See the NOTICE file\n+ *  distributed with this work for additional information\n+ *  regarding copyright ownership.  The ASF licenses this file\n+ *  to you under the Apache License, Version 2.0 (the\n+ *  \"License\"); you may not use this file except in compliance\n+ *  with the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ *  Unless required by applicable law or agreed to in writing,\n+ *  software distributed under the License is distributed on an\n+ *  \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n+ *  KIND, either express or implied.  See the License for the\n+ *  specific language governing permissions and limitations\n+ *  under the License.\n+ */\n+\n+package org.apache.parquet.hadoop.util;\n+\n+import org.apache.hadoop.fs.FSDataInputStream;\n+import org.apache.parquet.Log;\n+import org.apache.parquet.io.ParquetDecodingException;\n+import org.apache.parquet.io.SeekableInputStream;\n+import java.io.IOException;\n+import java.lang.reflect.Constructor;\n+import java.lang.reflect.InvocationTargetException;\n+import java.nio.ByteBuffer;\n+\n+public class HadoopStreams {\n+\n+  private static final Log LOG = Log.getLog(SeekableInputStream.class);\n+\n+  private static final Class<?> byteBufferReadableClass;\n+\n+  private static final Constructor<SeekableInputStream> h2SeekableConstructor;\n+\n+  static {\n+    byteBufferReadableClass = getReadableClass();\n+    h2SeekableConstructor = getH2SeekableConstructor();\n+  }\n+\n+  public static SeekableInputStream wrap(FSDataInputStream stream) {\n+    if (byteBufferReadableClass != null && h2SeekableConstructor != null &&\n+        byteBufferReadableClass.isInstance(stream) &&\n+        supportsByteBufferReads(stream)) {\n+      try {\n+        return h2SeekableConstructor.newInstance(stream);\n+      } catch (InstantiationException e) {\n+        LOG.warn(\"Could not instantiate H2SeekableInputStream, falling back to byte array reads\", e);\n+        return new H1SeekableInputStream(stream);\n+      } catch (IllegalAccessException e) {\n+        LOG.warn(\"Could not instantiate H2SeekableInputStream, falling back to byte array reads\", e);\n+        return new H1SeekableInputStream(stream);\n+      } catch (InvocationTargetException e) {\n+        throw new ParquetDecodingException(\n+            \"Could not instantiate H2SeekableInputStream\", e.getTargetException());\n+      }\n+    } else {\n+      return new H1SeekableInputStream(stream);\n+    }\n+  }\n+\n+  private static final ByteBuffer ZERO_LEN_BYTE_BUFFER = ByteBuffer.wrap(new byte[0]);\n+\n+  private static boolean supportsByteBufferReads(FSDataInputStream stream) {\n+    // FSDataInputStream implements ByteBufferReadable, but may throw\n+    // UnsupportedOperationException if it isn't actually supported. This tests\n+    // whether the method throws by trying to read a 0-length buffer, which has\n+    // no effect on the stream when it is supported.\n+    try {\n+      stream.read(ZERO_LEN_BYTE_BUFFER);\n+      return true;\n+    } catch (UnsupportedOperationException e) {\n+      return false;\n+    } catch (IOException e) {\n+      return false;\n+    }\n+  }\n+\n+  private static Class<?> getReadableClass() {\n+    try {\n+      return Class.forName(\"org.apache.hadoop.fs.ByteBufferReadable\");\n+    } catch (ClassNotFoundException e) {\n+      return null;\n+    } catch (NoClassDefFoundError e) {\n+      return null;\n+    }\n+  }\n+\n+  @SuppressWarnings(\"unchecked\")\n+  private static Class<SeekableInputStream> getH2SeekableClass() {\n+    try {\n+      return (Class<SeekableInputStream>) Class.forName(\n+          \"org.apache.parquet.hadoop.util.H2SeekableInputStream\");","path":"parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopStreams.java","position":96,"original_position":96,"commit_id":"e43d54d36bceaee956227e4d366858816b1291bd","original_commit_id":"e43d54d36bceaee956227e4d366858816b1291bd","user":{"login":"piyushnarang","id":9885908,"avatar_url":"https://avatars.githubusercontent.com/u/9885908?v=3","gravatar_id":"","url":"https://api.github.com/users/piyushnarang","html_url":"https://github.com/piyushnarang","followers_url":"https://api.github.com/users/piyushnarang/followers","following_url":"https://api.github.com/users/piyushnarang/following{/other_user}","gists_url":"https://api.github.com/users/piyushnarang/gists{/gist_id}","starred_url":"https://api.github.com/users/piyushnarang/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/piyushnarang/subscriptions","organizations_url":"https://api.github.com/users/piyushnarang/orgs","repos_url":"https://api.github.com/users/piyushnarang/repos","events_url":"https://api.github.com/users/piyushnarang/events{/privacy}","received_events_url":"https://api.github.com/users/piyushnarang/received_events","type":"User","site_admin":false},"body":"pull out as a constant","created_at":"2016-06-29T20:32:52Z","updated_at":"2016-06-29T20:32:52Z","html_url":"https://github.com/apache/parquet-mr/pull/349#discussion_r69021725","pull_request_url":"https://api.github.com/repos/apache/parquet-mr/pulls/349","_links":{"self":{"href":"https://api.github.com/repos/apache/parquet-mr/pulls/comments/69021725"},"html":{"href":"https://github.com/apache/parquet-mr/pull/349#discussion_r69021725"},"pull_request":{"href":"https://api.github.com/repos/apache/parquet-mr/pulls/349"}}},"pull_request":{"url":"https://api.github.com/repos/apache/parquet-mr/pulls/349","id":75573401,"html_url":"https://github.com/apache/parquet-mr/pull/349","diff_url":"https://github.com/apache/parquet-mr/pull/349.diff","patch_url":"https://github.com/apache/parquet-mr/pull/349.patch","issue_url":"https://api.github.com/repos/apache/parquet-mr/issues/349","number":349,"state":"open","locked":false,"title":"PARQUET-400: Replace CompatibilityUtil with SeekableInputStream.","user":{"login":"rdblue","id":87915,"avatar_url":"https://avatars.githubusercontent.com/u/87915?v=3","gravatar_id":"","url":"https://api.github.com/users/rdblue","html_url":"https://github.com/rdblue","followers_url":"https://api.github.com/users/rdblue/followers","following_url":"https://api.github.com/users/rdblue/following{/other_user}","gists_url":"https://api.github.com/users/rdblue/gists{/gist_id}","starred_url":"https://api.github.com/users/rdblue/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rdblue/subscriptions","organizations_url":"https://api.github.com/users/rdblue/orgs","repos_url":"https://api.github.com/users/rdblue/repos","events_url":"https://api.github.com/users/rdblue/events{/privacy}","received_events_url":"https://api.github.com/users/rdblue/received_events","type":"User","site_admin":false},"body":"This fixes PARQUET-400 by replacing `CompatibilityUtil` with `SeekableInputStream` that's implemented for hadoop-1 and hadoop-2. The benefit of this approach is that `SeekableInputStream` can be used for non-Hadoop file systems in the future.\r\n\r\nThis also changes the default Hadoop version to Hadoop-2. The library is still compatible with Hadoop 1.x, but this makes building Hadoop-2 classes, like `H2SeekableInputStream`, much easier and removes the need for multiple hadoop versions during compilation.","created_at":"2016-06-29T00:50:46Z","updated_at":"2016-06-29T20:32:52Z","closed_at":null,"merged_at":null,"merge_commit_sha":"05f9e8d8ae6d9f124ba57d34c4eea5e66b8911c7","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/parquet-mr/pulls/349/commits","review_comments_url":"https://api.github.com/repos/apache/parquet-mr/pulls/349/comments","review_comment_url":"https://api.github.com/repos/apache/parquet-mr/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/parquet-mr/issues/349/comments","statuses_url":"https://api.github.com/repos/apache/parquet-mr/statuses/e43d54d36bceaee956227e4d366858816b1291bd","head":{"label":"rdblue:PARQUET-400-byte-buffers","ref":"PARQUET-400-byte-buffers","sha":"e43d54d36bceaee956227e4d366858816b1291bd","user":{"login":"rdblue","id":87915,"avatar_url":"https://avatars.githubusercontent.com/u/87915?v=3","gravatar_id":"","url":"https://api.github.com/users/rdblue","html_url":"https://github.com/rdblue","followers_url":"https://api.github.com/users/rdblue/followers","following_url":"https://api.github.com/users/rdblue/following{/other_user}","gists_url":"https://api.github.com/users/rdblue/gists{/gist_id}","starred_url":"https://api.github.com/users/rdblue/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rdblue/subscriptions","organizations_url":"https://api.github.com/users/rdblue/orgs","repos_url":"https://api.github.com/users/rdblue/repos","events_url":"https://api.github.com/users/rdblue/events{/privacy}","received_events_url":"https://api.github.com/users/rdblue/received_events","type":"User","site_admin":false},"repo":{"id":21755687,"name":"parquet-mr","full_name":"rdblue/parquet-mr","owner":{"login":"rdblue","id":87915,"avatar_url":"https://avatars.githubusercontent.com/u/87915?v=3","gravatar_id":"","url":"https://api.github.com/users/rdblue","html_url":"https://github.com/rdblue","followers_url":"https://api.github.com/users/rdblue/followers","following_url":"https://api.github.com/users/rdblue/following{/other_user}","gists_url":"https://api.github.com/users/rdblue/gists{/gist_id}","starred_url":"https://api.github.com/users/rdblue/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rdblue/subscriptions","organizations_url":"https://api.github.com/users/rdblue/orgs","repos_url":"https://api.github.com/users/rdblue/repos","events_url":"https://api.github.com/users/rdblue/events{/privacy}","received_events_url":"https://api.github.com/users/rdblue/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/rdblue/parquet-mr","description":"Mirror of Apache Parquet","fork":true,"url":"https://api.github.com/repos/rdblue/parquet-mr","forks_url":"https://api.github.com/repos/rdblue/parquet-mr/forks","keys_url":"https://api.github.com/repos/rdblue/parquet-mr/keys{/key_id}","collaborators_url":"https://api.github.com/repos/rdblue/parquet-mr/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/rdblue/parquet-mr/teams","hooks_url":"https://api.github.com/repos/rdblue/parquet-mr/hooks","issue_events_url":"https://api.github.com/repos/rdblue/parquet-mr/issues/events{/number}","events_url":"https://api.github.com/repos/rdblue/parquet-mr/events","assignees_url":"https://api.github.com/repos/rdblue/parquet-mr/assignees{/user}","branches_url":"https://api.github.com/repos/rdblue/parquet-mr/branches{/branch}","tags_url":"https://api.github.com/repos/rdblue/parquet-mr/tags","blobs_url":"https://api.github.com/repos/rdblue/parquet-mr/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/rdblue/parquet-mr/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/rdblue/parquet-mr/git/refs{/sha}","trees_url":"https://api.github.com/repos/rdblue/parquet-mr/git/trees{/sha}","statuses_url":"https://api.github.com/repos/rdblue/parquet-mr/statuses/{sha}","languages_url":"https://api.github.com/repos/rdblue/parquet-mr/languages","stargazers_url":"https://api.github.com/repos/rdblue/parquet-mr/stargazers","contributors_url":"https://api.github.com/repos/rdblue/parquet-mr/contributors","subscribers_url":"https://api.github.com/repos/rdblue/parquet-mr/subscribers","subscription_url":"https://api.github.com/repos/rdblue/parquet-mr/subscription","commits_url":"https://api.github.com/repos/rdblue/parquet-mr/commits{/sha}","git_commits_url":"https://api.github.com/repos/rdblue/parquet-mr/git/commits{/sha}","comments_url":"https://api.github.com/repos/rdblue/parquet-mr/comments{/number}","issue_comment_url":"https://api.github.com/repos/rdblue/parquet-mr/issues/comments{/number}","contents_url":"https://api.github.com/repos/rdblue/parquet-mr/contents/{+path}","compare_url":"https://api.github.com/repos/rdblue/parquet-mr/compare/{base}...{head}","merges_url":"https://api.github.com/repos/rdblue/parquet-mr/merges","archive_url":"https://api.github.com/repos/rdblue/parquet-mr/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/rdblue/parquet-mr/downloads","issues_url":"https://api.github.com/repos/rdblue/parquet-mr/issues{/number}","pulls_url":"https://api.github.com/repos/rdblue/parquet-mr/pulls{/number}","milestones_url":"https://api.github.com/repos/rdblue/parquet-mr/milestones{/number}","notifications_url":"https://api.github.com/repos/rdblue/parquet-mr/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/rdblue/parquet-mr/labels{/name}","releases_url":"https://api.github.com/repos/rdblue/parquet-mr/releases{/id}","deployments_url":"https://api.github.com/repos/rdblue/parquet-mr/deployments","created_at":"2014-07-12T00:08:35Z","updated_at":"2015-04-29T00:20:04Z","pushed_at":"2016-06-29T19:40:31Z","git_url":"git://github.com/rdblue/parquet-mr.git","ssh_url":"git@github.com:rdblue/parquet-mr.git","clone_url":"https://github.com/rdblue/parquet-mr.git","svn_url":"https://github.com/rdblue/parquet-mr","homepage":null,"size":14535,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":1,"mirror_url":null,"open_issues_count":0,"forks":1,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"da69d4b764f4d13d38a4f7fe7462ef0c7d17c619","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":20675636,"name":"parquet-mr","full_name":"apache/parquet-mr","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/parquet-mr","description":"Mirror of Apache Parquet","fork":false,"url":"https://api.github.com/repos/apache/parquet-mr","forks_url":"https://api.github.com/repos/apache/parquet-mr/forks","keys_url":"https://api.github.com/repos/apache/parquet-mr/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/parquet-mr/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/parquet-mr/teams","hooks_url":"https://api.github.com/repos/apache/parquet-mr/hooks","issue_events_url":"https://api.github.com/repos/apache/parquet-mr/issues/events{/number}","events_url":"https://api.github.com/repos/apache/parquet-mr/events","assignees_url":"https://api.github.com/repos/apache/parquet-mr/assignees{/user}","branches_url":"https://api.github.com/repos/apache/parquet-mr/branches{/branch}","tags_url":"https://api.github.com/repos/apache/parquet-mr/tags","blobs_url":"https://api.github.com/repos/apache/parquet-mr/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/parquet-mr/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/parquet-mr/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/parquet-mr/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/parquet-mr/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/parquet-mr/languages","stargazers_url":"https://api.github.com/repos/apache/parquet-mr/stargazers","contributors_url":"https://api.github.com/repos/apache/parquet-mr/contributors","subscribers_url":"https://api.github.com/repos/apache/parquet-mr/subscribers","subscription_url":"https://api.github.com/repos/apache/parquet-mr/subscription","commits_url":"https://api.github.com/repos/apache/parquet-mr/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/parquet-mr/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/parquet-mr/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/parquet-mr/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/parquet-mr/contents/{+path}","compare_url":"https://api.github.com/repos/apache/parquet-mr/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/parquet-mr/merges","archive_url":"https://api.github.com/repos/apache/parquet-mr/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/parquet-mr/downloads","issues_url":"https://api.github.com/repos/apache/parquet-mr/issues{/number}","pulls_url":"https://api.github.com/repos/apache/parquet-mr/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/parquet-mr/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/parquet-mr/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/parquet-mr/labels{/name}","releases_url":"https://api.github.com/repos/apache/parquet-mr/releases{/id}","deployments_url":"https://api.github.com/repos/apache/parquet-mr/deployments","created_at":"2014-06-10T07:00:07Z","updated_at":"2016-06-29T20:32:13Z","pushed_at":"2016-06-29T19:40:31Z","git_url":"git://github.com/apache/parquet-mr.git","ssh_url":"git@github.com:apache/parquet-mr.git","clone_url":"https://github.com/apache/parquet-mr.git","svn_url":"https://github.com/apache/parquet-mr","homepage":null,"size":17713,"stargazers_count":252,"watchers_count":252,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":229,"mirror_url":"git://git.apache.org/parquet-mr.git","open_issues_count":54,"forks":229,"open_issues":54,"watchers":252,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/parquet-mr/pulls/349"},"html":{"href":"https://github.com/apache/parquet-mr/pull/349"},"issue":{"href":"https://api.github.com/repos/apache/parquet-mr/issues/349"},"comments":{"href":"https://api.github.com/repos/apache/parquet-mr/issues/349/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/parquet-mr/pulls/349/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/parquet-mr/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/parquet-mr/pulls/349/commits"},"statuses":{"href":"https://api.github.com/repos/apache/parquet-mr/statuses/e43d54d36bceaee956227e4d366858816b1291bd"}}}}
{"actor":{"display_login":"piyushnarang"}}