{"action":"created","comment":{"url":"https://api.github.com/repos/apache/incubator-pinot/pulls/comments/260798088","pull_request_review_id":208583850,"id":260798088,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDI2MDc5ODA4OA==","diff_hunk":"@@ -18,142 +18,139 @@\n  */\n package org.apache.pinot.broker.broker.helix;\n \n-import java.util.ArrayList;\n+import com.google.common.base.Preconditions;\n import java.util.List;\n-import java.util.concurrent.LinkedBlockingQueue;\n+import java.util.Map;\n+import java.util.concurrent.ConcurrentHashMap;\n import java.util.concurrent.TimeUnit;\n-import org.apache.commons.lang3.tuple.ImmutablePair;\n-import org.apache.commons.lang3.tuple.Pair;\n+import org.apache.helix.HelixConstants.ChangeType;\n import org.apache.helix.NotificationContext;\n+import org.apache.helix.api.listeners.BatchMode;\n import org.apache.helix.api.listeners.ExternalViewChangeListener;\n import org.apache.helix.api.listeners.InstanceConfigChangeListener;\n import org.apache.helix.api.listeners.LiveInstanceChangeListener;\n+import org.apache.helix.api.listeners.PreFetch;\n import org.apache.helix.model.ExternalView;\n import org.apache.helix.model.InstanceConfig;\n import org.apache.helix.model.LiveInstance;\n-import org.apache.pinot.broker.queryquota.TableQueryQuotaManager;\n-import org.apache.pinot.broker.routing.HelixExternalViewBasedRouting;\n+import org.apache.pinot.common.metrics.BrokerMeter;\n import org.apache.pinot.common.metrics.BrokerMetrics;\n import org.apache.pinot.common.metrics.BrokerTimer;\n import org.slf4j.Logger;\n import org.slf4j.LoggerFactory;\n \n \n /**\n- * Manages the interactions between Helix cluster changes, the routing table and the connection pool.\n+ * The {@code ClusterChangeMediator} handles the changes from Helix cluster.\n+ * <p>\n+ * <p>If there is no change callback in 1 hour, proactively check changes so that the changes are getting processed even\n+ * when callbacks stop working.\n+ * <p>NOTE: disable Helix batch-mode and perform deduplication in this class.\n+ * <p>NOTE: disable Helix pre-fetch to reduce the ZK accesses.\n  */\n-public class ClusterChangeMediator implements LiveInstanceChangeListener, ExternalViewChangeListener, InstanceConfigChangeListener {\n+@BatchMode(enabled = false)\n+@PreFetch(enabled = false)\n+public class ClusterChangeMediator implements ExternalViewChangeListener, InstanceConfigChangeListener, LiveInstanceChangeListener {\n   private static final Logger LOGGER = LoggerFactory.getLogger(ClusterChangeMediator.class);\n-  private final HelixExternalViewBasedRouting _helixExternalViewBasedRouting;\n-  private final TableQueryQuotaManager _tableQueryQuotaManager;\n \n-  private enum UpdateType {\n-    EXTERNAL_VIEW, INSTANCE_CONFIG\n-  }\n+  // Add 1 second interval between change checks to deduplicate multiple changes of the same type\n+  private static final long CHANGE_CHECK_INTERVAL_MS = 1000L;\n+  // If no change got for 1 hour, proactively check changes\n+  private static final long PROACTIVE_CHANGE_CHECK_INTERVAL_MS = 3600 * 1000L;\n+\n+  private final Map<ChangeType, ClusterChangeHandler> _changeHandlerMap;\n+  private final Map<ChangeType, Long> _lastChangeTimeMap = new ConcurrentHashMap<>();\n+  private final Map<ChangeType, Long> _lastProcessTimeMap = new ConcurrentHashMap<>();\n \n-  private final LinkedBlockingQueue<Pair<UpdateType, Long>> _clusterChangeQueue = new LinkedBlockingQueue<>(1000);\n+  private final Thread _clusterChangeHandlingThread;\n \n-  private Thread _deferredClusterUpdater = null;\n+  public ClusterChangeMediator(Map<ChangeType, ClusterChangeHandler> changeHandlerMap, BrokerMetrics brokerMetrics) {\n+    _changeHandlerMap = changeHandlerMap;\n \n-  public ClusterChangeMediator(HelixExternalViewBasedRouting helixExternalViewBasedRouting,\n-      TableQueryQuotaManager tableQueryQuotaManager, final BrokerMetrics brokerMetrics) {\n-    _helixExternalViewBasedRouting = helixExternalViewBasedRouting;\n-    _tableQueryQuotaManager = tableQueryQuotaManager;\n+    // Initialize last process time map\n+    long initTime = System.currentTimeMillis();\n+    for (ChangeType changeType : changeHandlerMap.keySet()) {\n+      _lastProcessTimeMap.put(changeType, initTime);\n+    }\n \n-    // Simple thread that polls every 10 seconds to check if there are any cluster updates to apply\n-    _deferredClusterUpdater = new Thread(\"Deferred cluster state updater\") {\n+    _clusterChangeHandlingThread = new Thread(\"ClusterChangeHandlingThread\") {\n       @Override\n       public void run() {\n         while (true) {\n           try {\n-            // Wait for at least one update\n-            Pair<UpdateType, Long> firstUpdate = _clusterChangeQueue.take();\n-\n-            // Update the queue time metrics\n-            long queueTime = System.currentTimeMillis() - firstUpdate.getValue();\n-            brokerMetrics.addTimedValue(BrokerTimer.ROUTING_TABLE_UPDATE_QUEUE_TIME, queueTime, TimeUnit.MILLISECONDS);\n-\n-            // Take all other updates also present\n-            List<Pair<UpdateType, Long>> allUpdates = new ArrayList<>();\n-            allUpdates.add(firstUpdate);\n-            _clusterChangeQueue.drainTo(allUpdates);\n-\n-            // Gather all update types\n-            boolean externalViewUpdated = false;\n-            boolean instanceConfigUpdated = false;\n-\n-            for (Pair<UpdateType, Long> update : allUpdates) {\n-              if (update.getKey() == UpdateType.EXTERNAL_VIEW) {\n-                externalViewUpdated = true;\n-              } else if (update.getKey() == UpdateType.INSTANCE_CONFIG) {\n-                instanceConfigUpdated = true;\n-              }\n-            }\n-\n-            if (externalViewUpdated) {\n-              try {\n-                _helixExternalViewBasedRouting.processExternalViewChange();\n-                _tableQueryQuotaManager.processQueryQuotaChange();\n-              } catch (Exception e) {\n-                LOGGER.warn(\"Caught exception while updating external view\", e);\n+            for (Map.Entry<ChangeType, ClusterChangeHandler> entry : _changeHandlerMap.entrySet()) {\n+              ChangeType changeType = entry.getKey();\n+              ClusterChangeHandler changeHandler = entry.getValue();\n+              long currentTime = System.currentTimeMillis();\n+              Long lastChangeTime = _lastChangeTimeMap.remove(changeType);\n+              if (lastChangeTime != null) {\n+                brokerMetrics.addTimedValue(BrokerTimer.CLUSTER_CHANGE_QUEUE_TIME, currentTime - lastChangeTime,\n+                    TimeUnit.MILLISECONDS);\n+                processClusterChange(changeType, changeHandler);\n+              } else {\n+                long lastProcessTime = _lastProcessTimeMap.get(changeType);\n+                if (currentTime - lastProcessTime > PROACTIVE_CHANGE_CHECK_INTERVAL_MS) {\n+                  LOGGER.info(\"Proactive check {} change\", changeType);\n+                  brokerMetrics.addMeteredGlobalValue(BrokerMeter.PROACTIVE_CLUSTER_CHANGE_CHECK, 1L);\n+                  processClusterChange(changeType, changeHandler);\n+                }\n               }\n             }\n \n-            if (instanceConfigUpdated) {\n-              try {\n-                _helixExternalViewBasedRouting.processInstanceConfigChange();\n-              } catch (Exception e) {\n-                LOGGER.warn(\"Caught exception while processing instance config\", e);\n-              }\n-            }\n+            // Add an interval between change checks to deduplicate multiple changes of the same type\n+            Thread.sleep(CHANGE_CHECK_INTERVAL_MS);\n           } catch (InterruptedException e) {\n-            LOGGER.warn(\"Was interrupted while waiting for a cluster change\", e);\n+            LOGGER.warn(\"Cluster change handling thread is interrupted, stopping the thread\");\n             break;\n+          } catch (Exception e) {\n+            LOGGER.error(\"Caught exception while handling changes\", e);\n           }\n         }\n-\n-        LOGGER.warn(\"Stopping deferred cluster state update thread\");\n-        _deferredClusterUpdater = null;\n       }\n     };\n+    _clusterChangeHandlingThread.start();\n+  }\n \n-    _deferredClusterUpdater.start();\n+  private void processClusterChange(ChangeType changeType, ClusterChangeHandler changeHandler) {\n+    long startTime = System.currentTimeMillis();\n+    LOGGER.info(\"Start processing {} change\", changeType);\n+    changeHandler.processClusterChange();\n+    long endTime = System.currentTimeMillis();\n+    LOGGER.info(\"Finish processing {} change in {}ms\", changeType, endTime - startTime);\n+    _lastProcessTimeMap.put(changeType, endTime);\n   }\n \n   @Override\n   public void onExternalViewChange(List<ExternalView> externalViewList, NotificationContext changeContext) {\n-    // If the deferred update thread is alive, defer the update\n-    if (_deferredClusterUpdater != null && _deferredClusterUpdater.isAlive()) {\n-      try {\n-        _clusterChangeQueue.put(new ImmutablePair<>(UpdateType.EXTERNAL_VIEW, System.currentTimeMillis()));\n-      } catch (InterruptedException e) {\n-        LOGGER.warn(\"Was interrupted while trying to add external view change to queue\", e);\n-      }\n-    } else {\n-      LOGGER.warn(\n-          \"Deferred cluster updater thread is null or stopped, not deferring external view routing table rebuild\");\n-      _helixExternalViewBasedRouting.processExternalViewChange();\n-      _tableQueryQuotaManager.processQueryQuotaChange();\n-    }\n+    Preconditions.checkState(externalViewList.isEmpty(), \"Helix pre-fetch should be disabled\");\n+    Preconditions.checkState(_clusterChangeHandlingThread.isAlive(), \"Cluster change handling thread is not alive\");\n+\n+    enqueueChange(ChangeType.EXTERNAL_VIEW);\n   }\n \n   @Override\n-  public void onInstanceConfigChange(List<InstanceConfig> instanceConfigs, NotificationContext context) {\n-    // If the deferred update thread is alive, defer the update\n-    if (_deferredClusterUpdater != null && _deferredClusterUpdater.isAlive()) {\n-      try {\n-        _clusterChangeQueue.put(new ImmutablePair<>(UpdateType.INSTANCE_CONFIG, System.currentTimeMillis()));\n-      } catch (InterruptedException e) {\n-        LOGGER.warn(\"Was interrupted while trying to add external view change to queue\", e);\n-      }\n-    } else {\n-      LOGGER.warn(\n-          \"Deferred cluster updater thread is null or stopped, not deferring instance config change notification\");\n-      _helixExternalViewBasedRouting.processInstanceConfigChange();\n-    }\n+  public void onInstanceConfigChange(List<InstanceConfig> instanceConfigs, NotificationContext changeContext) {\n+    Preconditions.checkState(instanceConfigs.isEmpty(), \"Helix pre-fetch should be disabled\");\n+    Preconditions.checkState(_clusterChangeHandlingThread.isAlive(), \"Cluster change handling thread is not alive\");","path":"pinot-broker/src/main/java/org/apache/pinot/broker/broker/helix/ClusterChangeMediator.java","position":206,"original_position":206,"commit_id":"6700ac590924e9ff66d0ef143b75b7f1378dae76","original_commit_id":"6700ac590924e9ff66d0ef143b75b7f1378dae76","user":{"login":"sunithabeeram","id":3123798,"node_id":"MDQ6VXNlcjMxMjM3OTg=","avatar_url":"https://avatars2.githubusercontent.com/u/3123798?v=4","gravatar_id":"","url":"https://api.github.com/users/sunithabeeram","html_url":"https://github.com/sunithabeeram","followers_url":"https://api.github.com/users/sunithabeeram/followers","following_url":"https://api.github.com/users/sunithabeeram/following{/other_user}","gists_url":"https://api.github.com/users/sunithabeeram/gists{/gist_id}","starred_url":"https://api.github.com/users/sunithabeeram/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sunithabeeram/subscriptions","organizations_url":"https://api.github.com/users/sunithabeeram/orgs","repos_url":"https://api.github.com/users/sunithabeeram/repos","events_url":"https://api.github.com/users/sunithabeeram/events{/privacy}","received_events_url":"https://api.github.com/users/sunithabeeram/received_events","type":"User","site_admin":false},"body":"Its not clear who handles this runtime exception and what the recovery path is. The earlier code processed the change inline.","created_at":"2019-02-27T15:21:23Z","updated_at":"2019-02-27T15:23:56Z","html_url":"https://github.com/apache/incubator-pinot/pull/3885#discussion_r260798088","pull_request_url":"https://api.github.com/repos/apache/incubator-pinot/pulls/3885","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/apache/incubator-pinot/pulls/comments/260798088"},"html":{"href":"https://github.com/apache/incubator-pinot/pull/3885#discussion_r260798088"},"pull_request":{"href":"https://api.github.com/repos/apache/incubator-pinot/pulls/3885"}}},"pull_request":{"url":"https://api.github.com/repos/apache/incubator-pinot/pulls/3885","id":256113485,"node_id":"MDExOlB1bGxSZXF1ZXN0MjU2MTEzNDg1","html_url":"https://github.com/apache/incubator-pinot/pull/3885","diff_url":"https://github.com/apache/incubator-pinot/pull/3885.diff","patch_url":"https://github.com/apache/incubator-pinot/pull/3885.patch","issue_url":"https://api.github.com/repos/apache/incubator-pinot/issues/3885","number":3885,"state":"open","locked":false,"title":"Actively check cluster changes if there is no callback for a long time","user":{"login":"Jackie-Jiang","id":17555551,"node_id":"MDQ6VXNlcjE3NTU1NTUx","avatar_url":"https://avatars3.githubusercontent.com/u/17555551?v=4","gravatar_id":"","url":"https://api.github.com/users/Jackie-Jiang","html_url":"https://github.com/Jackie-Jiang","followers_url":"https://api.github.com/users/Jackie-Jiang/followers","following_url":"https://api.github.com/users/Jackie-Jiang/following{/other_user}","gists_url":"https://api.github.com/users/Jackie-Jiang/gists{/gist_id}","starred_url":"https://api.github.com/users/Jackie-Jiang/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Jackie-Jiang/subscriptions","organizations_url":"https://api.github.com/users/Jackie-Jiang/orgs","repos_url":"https://api.github.com/users/Jackie-Jiang/repos","events_url":"https://api.github.com/users/Jackie-Jiang/events{/privacy}","received_events_url":"https://api.github.com/users/Jackie-Jiang/received_events","type":"User","site_admin":false},"body":"We encountered a site issue recently, and is suspecting that Helix callback for cluster changes is not working, probably because ZK re-connection.\nThis PR enabled the proactive cluster change check if there is no callback for 1 hour.\n\nChanges include:\n1. Rewrite ClusterChangeMediator to proactively perform cluster change check\n2. Disable the Helix batch-mode and perform deduplication in ClusterChangeMediator\n3. Disable the Helix pre-fetch to reduce the ZK accesses\n4. Add interface ClusterChangeHandler for general cluster change handle\n5. Add ClusterChangeHandler implementation: ExternalViewChangeHandler, InstanceConfigChangeHandler, LiveInstanceChangeHandler\n6. Add metrics to track CLUSTER_CHANGE_QUEUE_TIME and PROACTIVE_CLUSTER_CHANGE_CHECK count","created_at":"2019-02-26T01:56:13Z","updated_at":"2019-02-27T15:23:56Z","closed_at":null,"merged_at":null,"merge_commit_sha":"79c4d63b1adbf9d62a56dd8c4147834646ac98c4","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/incubator-pinot/pulls/3885/commits","review_comments_url":"https://api.github.com/repos/apache/incubator-pinot/pulls/3885/comments","review_comment_url":"https://api.github.com/repos/apache/incubator-pinot/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/incubator-pinot/issues/3885/comments","statuses_url":"https://api.github.com/repos/apache/incubator-pinot/statuses/6700ac590924e9ff66d0ef143b75b7f1378dae76","head":{"label":"apache:external_view_update","ref":"external_view_update","sha":"6700ac590924e9ff66d0ef143b75b7f1378dae76","user":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":19961085,"node_id":"MDEwOlJlcG9zaXRvcnkxOTk2MTA4NQ==","name":"incubator-pinot","full_name":"apache/incubator-pinot","private":false,"owner":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/apache/incubator-pinot","description":"Apache Pinot (Incubating) - A realtime distributed OLAP datastore","fork":false,"url":"https://api.github.com/repos/apache/incubator-pinot","forks_url":"https://api.github.com/repos/apache/incubator-pinot/forks","keys_url":"https://api.github.com/repos/apache/incubator-pinot/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/incubator-pinot/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/incubator-pinot/teams","hooks_url":"https://api.github.com/repos/apache/incubator-pinot/hooks","issue_events_url":"https://api.github.com/repos/apache/incubator-pinot/issues/events{/number}","events_url":"https://api.github.com/repos/apache/incubator-pinot/events","assignees_url":"https://api.github.com/repos/apache/incubator-pinot/assignees{/user}","branches_url":"https://api.github.com/repos/apache/incubator-pinot/branches{/branch}","tags_url":"https://api.github.com/repos/apache/incubator-pinot/tags","blobs_url":"https://api.github.com/repos/apache/incubator-pinot/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/incubator-pinot/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/incubator-pinot/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/incubator-pinot/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/incubator-pinot/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/incubator-pinot/languages","stargazers_url":"https://api.github.com/repos/apache/incubator-pinot/stargazers","contributors_url":"https://api.github.com/repos/apache/incubator-pinot/contributors","subscribers_url":"https://api.github.com/repos/apache/incubator-pinot/subscribers","subscription_url":"https://api.github.com/repos/apache/incubator-pinot/subscription","commits_url":"https://api.github.com/repos/apache/incubator-pinot/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/incubator-pinot/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/incubator-pinot/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/incubator-pinot/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/incubator-pinot/contents/{+path}","compare_url":"https://api.github.com/repos/apache/incubator-pinot/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/incubator-pinot/merges","archive_url":"https://api.github.com/repos/apache/incubator-pinot/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/incubator-pinot/downloads","issues_url":"https://api.github.com/repos/apache/incubator-pinot/issues{/number}","pulls_url":"https://api.github.com/repos/apache/incubator-pinot/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/incubator-pinot/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/incubator-pinot/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/incubator-pinot/labels{/name}","releases_url":"https://api.github.com/repos/apache/incubator-pinot/releases{/id}","deployments_url":"https://api.github.com/repos/apache/incubator-pinot/deployments","created_at":"2014-05-19T23:27:48Z","updated_at":"2019-02-27T12:42:10Z","pushed_at":"2019-02-27T05:42:23Z","git_url":"git://github.com/apache/incubator-pinot.git","ssh_url":"git@github.com:apache/incubator-pinot.git","clone_url":"https://github.com/apache/incubator-pinot.git","svn_url":"https://github.com/apache/incubator-pinot","homepage":"https://pinot.apache.org","size":164046,"stargazers_count":2151,"watchers_count":2151,"language":"Java","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":383,"mirror_url":null,"archived":false,"open_issues_count":234,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":383,"open_issues":234,"watchers":2151,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"9269f5e02cfd868bd9448c6c4df9107e5776e60c","user":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":19961085,"node_id":"MDEwOlJlcG9zaXRvcnkxOTk2MTA4NQ==","name":"incubator-pinot","full_name":"apache/incubator-pinot","private":false,"owner":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/apache/incubator-pinot","description":"Apache Pinot (Incubating) - A realtime distributed OLAP datastore","fork":false,"url":"https://api.github.com/repos/apache/incubator-pinot","forks_url":"https://api.github.com/repos/apache/incubator-pinot/forks","keys_url":"https://api.github.com/repos/apache/incubator-pinot/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/incubator-pinot/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/incubator-pinot/teams","hooks_url":"https://api.github.com/repos/apache/incubator-pinot/hooks","issue_events_url":"https://api.github.com/repos/apache/incubator-pinot/issues/events{/number}","events_url":"https://api.github.com/repos/apache/incubator-pinot/events","assignees_url":"https://api.github.com/repos/apache/incubator-pinot/assignees{/user}","branches_url":"https://api.github.com/repos/apache/incubator-pinot/branches{/branch}","tags_url":"https://api.github.com/repos/apache/incubator-pinot/tags","blobs_url":"https://api.github.com/repos/apache/incubator-pinot/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/incubator-pinot/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/incubator-pinot/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/incubator-pinot/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/incubator-pinot/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/incubator-pinot/languages","stargazers_url":"https://api.github.com/repos/apache/incubator-pinot/stargazers","contributors_url":"https://api.github.com/repos/apache/incubator-pinot/contributors","subscribers_url":"https://api.github.com/repos/apache/incubator-pinot/subscribers","subscription_url":"https://api.github.com/repos/apache/incubator-pinot/subscription","commits_url":"https://api.github.com/repos/apache/incubator-pinot/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/incubator-pinot/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/incubator-pinot/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/incubator-pinot/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/incubator-pinot/contents/{+path}","compare_url":"https://api.github.com/repos/apache/incubator-pinot/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/incubator-pinot/merges","archive_url":"https://api.github.com/repos/apache/incubator-pinot/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/incubator-pinot/downloads","issues_url":"https://api.github.com/repos/apache/incubator-pinot/issues{/number}","pulls_url":"https://api.github.com/repos/apache/incubator-pinot/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/incubator-pinot/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/incubator-pinot/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/incubator-pinot/labels{/name}","releases_url":"https://api.github.com/repos/apache/incubator-pinot/releases{/id}","deployments_url":"https://api.github.com/repos/apache/incubator-pinot/deployments","created_at":"2014-05-19T23:27:48Z","updated_at":"2019-02-27T12:42:10Z","pushed_at":"2019-02-27T05:42:23Z","git_url":"git://github.com/apache/incubator-pinot.git","ssh_url":"git@github.com:apache/incubator-pinot.git","clone_url":"https://github.com/apache/incubator-pinot.git","svn_url":"https://github.com/apache/incubator-pinot","homepage":"https://pinot.apache.org","size":164046,"stargazers_count":2151,"watchers_count":2151,"language":"Java","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":383,"mirror_url":null,"archived":false,"open_issues_count":234,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":383,"open_issues":234,"watchers":2151,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/incubator-pinot/pulls/3885"},"html":{"href":"https://github.com/apache/incubator-pinot/pull/3885"},"issue":{"href":"https://api.github.com/repos/apache/incubator-pinot/issues/3885"},"comments":{"href":"https://api.github.com/repos/apache/incubator-pinot/issues/3885/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/incubator-pinot/pulls/3885/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/incubator-pinot/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/incubator-pinot/pulls/3885/commits"},"statuses":{"href":"https://api.github.com/repos/apache/incubator-pinot/statuses/6700ac590924e9ff66d0ef143b75b7f1378dae76"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"sunithabeeram"}}