{"action":"created","comment":{"url":"https://api.github.com/repos/linkedin/gobblin/pulls/comments/70534779","id":70534779,"diff_hunk":"@@ -0,0 +1,666 @@\n+/*\n+ * Copyright (C) 2014-2016 LinkedIn Corp. All rights reserved.\n+ *\n+ * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use\n+ * this file except in compliance with the License. You may obtain a copy of the\n+ * License at  http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software distributed\n+ * under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n+ * CONDITIONS OF ANY KIND, either express or implied.\n+ */\n+package gobblin.aws;\n+\n+import java.io.File;\n+import java.io.IOException;\n+import java.util.Arrays;\n+import java.util.Collections;\n+import java.util.Date;\n+import java.util.List;\n+import java.util.UUID;\n+import java.util.concurrent.TimeUnit;\n+import java.util.concurrent.TimeoutException;\n+\n+import org.apache.commons.io.FileUtils;\n+import org.apache.commons.mail.EmailException;\n+import org.apache.helix.Criteria;\n+import org.apache.helix.HelixManager;\n+import org.apache.helix.HelixManagerFactory;\n+import org.apache.helix.InstanceType;\n+import org.apache.helix.messaging.AsyncCallback;\n+import org.apache.helix.model.Message;\n+import org.slf4j.Logger;\n+import org.slf4j.LoggerFactory;\n+\n+import com.amazonaws.regions.Region;\n+import com.amazonaws.regions.Regions;\n+import com.amazonaws.services.autoscaling.model.AutoScalingGroup;\n+import com.amazonaws.services.autoscaling.model.BlockDeviceMapping;\n+import com.amazonaws.services.autoscaling.model.InstanceMonitoring;\n+import com.amazonaws.services.autoscaling.model.Tag;\n+import com.amazonaws.services.autoscaling.model.TagDescription;\n+import com.amazonaws.services.ec2.model.AvailabilityZone;\n+import com.amazonaws.services.ec2.model.Instance;\n+import com.google.common.annotations.VisibleForTesting;\n+import com.google.common.base.Optional;\n+import com.google.common.base.Throwables;\n+import com.google.common.collect.Lists;\n+import com.google.common.eventbus.EventBus;\n+import com.google.common.io.Closer;\n+import com.google.common.util.concurrent.Service;\n+import com.google.common.util.concurrent.ServiceManager;\n+import com.typesafe.config.Config;\n+import com.typesafe.config.ConfigFactory;\n+\n+import gobblin.annotation.Alpha;\n+import gobblin.cluster.GobblinClusterConfigurationKeys;\n+import gobblin.cluster.GobblinClusterUtils;\n+import gobblin.cluster.HelixMessageSubTypes;\n+import gobblin.cluster.HelixUtils;\n+import gobblin.util.ConfigUtils;\n+import gobblin.util.EmailUtils;\n+import static gobblin.aws.GobblinAWSUtils.*;\n+\n+\n+/**\n+ * A client driver to launch Gobblin as an AWS Cluster.\n+ *\n+ * <p>\n+ *   This class upon starting, will check if there is an AWS Cluster that is already running and\n+ *   it is able to reconnect to. More specifically, it checks if an cluster with the same cluster name\n+ *   exists and can be reconnected to i.e. if the cluster has not completed yet. If so, it simply starts\n+ *   monitoring that cluster.\n+ * </p>\n+ *\n+ * <p>\n+ *   On the other hand, if there's no such a reconnectable AWS cluster, This class will launch a new AWS\n+ *   cluster and start the {@link GobblinAWSClusterMaster}. It also persists the new cluster details so it\n+ *   is able to reconnect to the AWS cluster if it is restarted for some reason.\n+ * </p>\n+ *\n+ * <p>\n+ *   If a shutdown signal is received, it sends a Helix\n+ *   {@link org.apache.helix.model.Message.MessageType#SCHEDULER_MSG} to the {@link GobblinAWSClusterMaster}\n+ *   asking it to shutdown. It also sends an email notification for the shutdown if\n+ *   {@link GobblinAWSConfigurationKeys#EMAIL_NOTIFICATION_ON_SHUTDOWN_KEY} is {@code true}.\n+ * </p>\n+ *\n+ * @author Abhishek Tiwari\n+ */\n+@Alpha\n+public class GobblinAWSClusterLauncher {\n+  private static final Logger LOGGER = LoggerFactory.getLogger(GobblinAWSClusterLauncher.class);\n+\n+  public static final String CLUSTER_NAME_ASG_TAG = \"ClusterName\";\n+  public static final String CLUSTER_ID_ASG_TAG = \"ClusterId\";\n+  public static final String ASG_TYPE_ASG_TAG = \"AsgType\";\n+  public static final String ASG_TYPE_MASTER = \"master\";\n+  public static final String ASG_TYPE_WORKERS = \"workers\";\n+\n+  public static final String MASTER_ASG_NAME_PREFIX = \"GobblinMasterASG_\";\n+  public static final String MASTER_LAUNCH_CONFIG_NAME_PREFIX = \"GobblinMasterLaunchConfig_\";\n+  public static final String WORKERS_ASG_NAME_PREFIX = \"GobblinWorkerASG_\";\n+  public static final String WORKERS_LAUNCH_CONFIG_PREFIX = \"GobblinWorkerLaunchConfig_\";\n+\n+  private final Config config;\n+\n+  private final HelixManager helixManager;\n+  private final EventBus eventBus = new EventBus(GobblinAWSClusterLauncher.class.getSimpleName());\n+  private volatile Optional<ServiceManager> serviceManager = Optional.absent();\n+  private AWSClusterSecurityManager awsClusterSecurityManager;\n+  private AWSSdkClient awsSdkClient;\n+\n+  private final Closer closer = Closer.create();\n+\n+  // AWS cluster meta\n+  private final String clusterName;\n+  private volatile Optional<String> clusterId = Optional.absent();\n+\n+  private volatile boolean stopped = false;\n+  private final boolean emailNotificationOnShutdown;\n+\n+  // AWS Gobblin cluster common config\n+  private final String awsRegion;\n+  private final String awsConfDir;\n+\n+  // AWS Gobblin Master Instance config\n+  private final String masterAmiId;\n+  private final String masterInstanceType;\n+  private final String masterJvmMemory;\n+\n+  // AWS Gobblin Worker Instance config\n+  private final String workerAmiId;\n+  private final String workerInstanceType;\n+  private final String workerJvmMemory;\n+  private final Integer minWorkers;\n+  private final Integer maxWorkers;\n+  private final Integer desiredWorkers;\n+\n+  private final Optional<String> masterJvmArgs;\n+  private final Optional<String> workerJvmArgs;\n+\n+  private String masterPublicIp;\n+\n+  private final String nfsParentDir;\n+  private final String masterJarsDir;\n+  private final String masterConfLocalDir;\n+  private final String masterS3ConfUri;\n+  private final String masterS3ConfFiles;\n+  private final String masterS3JarsUri;\n+  private final String masterS3JarsFiles;\n+\n+  private final String workerJarsDir;\n+  private final String workerConfLocalDir;\n+  private final String workerS3ConfUri;\n+  private final String workerS3ConfFiles;\n+  private final String workerS3JarsUri;\n+  private final String workerS3JarsFiles;\n+\n+  private final String libJarsDir;\n+  private final String sinkLogRootDir;\n+  private final String appWorkDir;\n+\n+  private String masterLaunchConfigName;\n+  private String masterAutoScalingGroupName;\n+\n+  private String workerLaunchConfigName;\n+  private String workerAutoScalingGroupName;\n+\n+  private final Optional<String> gobblinVersion;\n+\n+\n+  public GobblinAWSClusterLauncher(Config config) throws IOException {","path":"gobblin-aws/src/main/java/gobblin/aws/GobblinAWSClusterLauncher.java","position":174,"original_position":172,"commit_id":"2216b005d801f9c69bb50e4535079872966554b3","original_commit_id":"8ead41415641119b3d42dbd9053f4b8573c1f842","user":{"login":"abti","id":69499,"avatar_url":"https://avatars.githubusercontent.com/u/69499?v=3","gravatar_id":"","url":"https://api.github.com/users/abti","html_url":"https://github.com/abti","followers_url":"https://api.github.com/users/abti/followers","following_url":"https://api.github.com/users/abti/following{/other_user}","gists_url":"https://api.github.com/users/abti/gists{/gist_id}","starred_url":"https://api.github.com/users/abti/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/abti/subscriptions","organizations_url":"https://api.github.com/users/abti/orgs","repos_url":"https://api.github.com/users/abti/repos","events_url":"https://api.github.com/users/abti/events{/privacy}","received_events_url":"https://api.github.com/users/abti/received_events","type":"User","site_admin":false},"body":"Changed to add defaults for most of the config values","created_at":"2016-07-12T22:33:09Z","updated_at":"2016-07-12T22:33:09Z","html_url":"https://github.com/linkedin/gobblin/pull/1088#discussion_r70534779","pull_request_url":"https://api.github.com/repos/linkedin/gobblin/pulls/1088","_links":{"self":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/comments/70534779"},"html":{"href":"https://github.com/linkedin/gobblin/pull/1088#discussion_r70534779"},"pull_request":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/1088"}}},"pull_request":{"url":"https://api.github.com/repos/linkedin/gobblin/pulls/1088","id":76390516,"html_url":"https://github.com/linkedin/gobblin/pull/1088","diff_url":"https://github.com/linkedin/gobblin/pull/1088.diff","patch_url":"https://github.com/linkedin/gobblin/pull/1088.patch","issue_url":"https://api.github.com/repos/linkedin/gobblin/issues/1088","number":1088,"state":"open","locked":false,"title":"Gobblin on AWS ","user":{"login":"abti","id":69499,"avatar_url":"https://avatars.githubusercontent.com/u/69499?v=3","gravatar_id":"","url":"https://api.github.com/users/abti","html_url":"https://github.com/abti","followers_url":"https://api.github.com/users/abti/followers","following_url":"https://api.github.com/users/abti/following{/other_user}","gists_url":"https://api.github.com/users/abti/gists{/gist_id}","starred_url":"https://api.github.com/users/abti/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/abti/subscriptions","organizations_url":"https://api.github.com/users/abti/orgs","repos_url":"https://api.github.com/users/abti/repos","events_url":"https://api.github.com/users/abti/events{/privacy}","received_events_url":"https://api.github.com/users/abti/received_events","type":"User","site_admin":false},"body":"## Introduction \r\n\r\nThis PR enables Gobblin to run in cluster mode on AWS via a new sub-module called `gobblin-aws` \r\n\r\n* `gobblin-aws` provides support to launch and run Gobblin cluster on AWS \r\n* `gobblin-aws` inherits from `gobblin-cluster` to make use of its abstraction wrapping Helix clustering capabilities \r\n* `gobblin-aws` performs all AWS resource management activities and provides for other critical cluster management capabilities not already provided by `gobblin-cluster` such as: High Availability, Elasticity and Failover\r\n\r\n## Architecture\r\n\r\n![gobblin-on-aws-architecture](https://cloud.githubusercontent.com/assets/69499/16608965/e094b320-4304-11e6-8352-a4d6a7031d25.png)\r\n\r\n### AWS Cluster Launcher\r\n\r\n`gobblin-aws` is launched using `GobblinAWSClusterLauncher` which upon bootup launches a few services: \r\n- `AWSClusterSecurityManager` to authenticate with AWS and refresh credentials at configured intervals\r\n- `AWSSdkClient` to coordinate with AWS and launch various resources \r\n\r\nAfter this, the `GobblinAWSClusterLauncher` computes the cloud-init scripts as well as AWS LaunchConfigurations and requests AWS to launch two `AutoScalingGroups*`:\r\n- Master `AutoScalingGroup` to launch and manage EC2 instances for `GobblinAWSClusterMaster` \r\n- Workers `AutoScalingGroup` to launch and manage EC2 instances for `GobblinAWSTaskRunner` \r\n\r\n### Master AutoScalingGroup\r\n\r\nWhen an EC2 instance under Master `AutoScalingGroup` boots up, it executes the master cloud-init script that:\r\n- First, creates a `NFS` server and setups work environment on the node (eg. creating all relevant directories) \r\n- Second, launches a java application to run `GobblinAWSClusterMaster` with relevant classpath and JVM arguments \r\n\r\nThe `GobblinAWSClusterMaster` upon bootup: \r\n- Registers itself as HelixController for the cluster\r\n- Launches a few services:\r\n- 1. `AWSJobConfigurationManager` that runs on configured frequency and polls for new or updated job configurations from S3\r\n- 2. `GobblinHelixJobScheduler` that schedules discovered new or updated jobs for execution\r\n\r\n### Workers AutoScalingGroup \r\n\r\nWhen n-number of EC2 instances under Workers `AutoScalingGroup` boot up, they execute worker's cloud-init script that: \r\n- First, mounts the `NFS` shared directory by master and setups work environment on the node\r\n- Second, launches a java application to run `GobblinAWSTaskRunner` with relevant classpath and JVM arguments \r\n\r\nThe `GobblinAWSTaskRunner` upon bootup: \r\n- Registers itself as a HelixParticipant in the cluster \r\n- Registers `GobblinHelixTaskFactory` to listen to Helix and process any tasks \r\n\r\n**AutoScalingGroup*** `AutoScalingGroup` is a service provided by AWS in which:\r\n- Users can specify minimum, maximum and desired number of instances\r\n- Scaling criteria based on which `AutoScalingGroup` automatically launches new instances or shuts them down to scale for load OR launch new instance if any instance dies. \r\n \r\nNote: \r\n1. When new instance launch, they execute the configured cloud-init script and hence register themselves as master or workers\r\n2. `AutoScalingGroup`s help us achieve HA and Elasticity for load \r\n\r\n### Execution Model  \r\nGobblin cluster on AWS makes use of Helix [Distributed Task Execution Framework](http://helix.apache.org/0.7.1-docs/recipes/task_dag_execution.html) to schedule and run jobs. \r\nFor each Gobblin job run: \r\n- The `GobblinHelixJobScheduler` on Master node starts a `GobblinHelixJobLauncher` that wraps the Gobblin job into a `GobblinHelixJob` and each Gobblin Task into a `GobblinHelixTask`, which implements the Helix's Task interface so Helix knows how to execute it \r\n- The `GobblinHelixJobLauncher` on Master node then submits the job to a Helix job queue named after the Gobblin job name, from which the Helix Distributed Task Execution Framework picks up the job and runs its tasks through the live participant worker nodes \r\n\r\n## Upcoming / Pending  \r\n- Replace `NFS` with `EFS` which recently went into GA \r\n- Multi master support for master HA \r\n- Support for spot instances \r\n- More hooks for auto scaling rules / parameters \r\n- Task execution JVM isolation (though this needs to be baked into `gobblin-cluster` )\r\n- `AMI` for AWS to bootup from a pre-baked Gobblin instance \r\n- Process watcher daemon running on all nodes (master and workers) to restart `GobblinAWSClusterMaster` and `GobblinAWSTaskRunner` process JVMs if they die \r\n\r\n## Running Gobblin-on-AWS \r\n\r\nFollow these steps to run Gobblin-on-AWS \r\n- Build Gobblin as you build today, and extract the distribution tar.gz \r\n- Modify `conf/aws/application.conf` as described later in `Configuration` section \r\n- Replace secret and access keys with dummy values in `application.conf` and upload it on S3 \r\n- Launch Gobblin cluster on AWS by running: `bin/gobblin-aws.sh start` \r\n \r\nA private key file with pem extension should be saved on the disk, you can use this to ssh into Master and Worker nodes as you would in any regular instance on AWS \r\n\r\n### Configuration \r\n\r\nGetting started with Gobblin on AWS is easy. Only the following configurations need to be changed in `conf/aws/application.conf` :\r\n```\r\n# Cluster name\r\ngobblin.aws.cluster.name=GobblinAWSPullRequest\r\n\r\n# Zookeeper connection string\r\n# Note: You can use the example below, but the example Zookeeper may or may not be available\r\n# Example: gobblin.cluster.zk.connection.string=\"ec2-54-149-223-115.us-west-2.compute.amazonaws.com:2181\"\r\ngobblin.cluster.zk.connection.string=\"ec2-54-149-223-115.us-west-2.compute.amazonaws.com:2181\"\r\n\r\n# AWS Credentials\r\ngobblin.aws.service.access=\"<AWS Access Key>\"\r\ngobblin.aws.service.secret=\"<AWS Secret Key>\"\r\n\r\n# Leave as-is for sample wikipedia pull job\r\n# -- OR --\r\n# Specify S3 URI for job configuration:\r\n# Steps to obtain this URI:\r\n# 1. Create a S3 folder in a bucket, say the bucket name is \"s3bucket\" and folder is \"/gobblin-confs/job-conf/\"\r\n# 2. Zip your pull files in a zip and upload to this folder. Say your zip file name is myPullFiles.zip\r\n# 3. Specify S3 URI for this zip file in the config below\r\n#    For the description above: \"https://s3-region.amazonaws.com/s3bucket/gobblin-confs/job-conf/myPullFiles.zip\"\r\ngobblin.aws.job.conf.s3.uri=\"https://s3-us-west-2.amazonaws.com/gobblin-libs/job-config/wikipedia.zip\"\r\n\r\n# Specify S3 URI for cluster configuration\r\n# Steps to obtain this URI:\r\n# 1. Create a S3 folder in a bucket, say the bucket name is \"s3bucket\" and folder is \"/gobblin-confs/cluster-conf/\"\r\n# 2. After editing this application.conf: upload this application.conf, log4j-aws.properties and quartz.properties to\r\n#    /s3bucket/gobblin-confs/cluster-conf/\r\n#    Note: Make sure to replace AWS credential values with dummy values.\r\n#          We don't read AWS credentials from application.conf in S3\r\n# 3. Specify S3 URI for this folder in the config below\r\n#    For the description above: \"https://s3-region.amazonaws.com/s3bucket/gobblin-confs/cluster-conf/\"\r\n# Example: gobblin.aws.master.s3.conf.uri=\"https://s3-us-west-2.amazonaws.com/gobblin-libs/cluster-config/\"\r\ngobblin.aws.master.s3.conf.uri=\"https://s3-region.amazonaws.com/s3bucket/gobblin-confs/cluster-conf/\" \r\n```\r\n\r\n## Notes\r\n- **Logs** for master and workers are saved in a common directory in `NFS` and are available in real time\r\n- All classes in this PR have been marked with `Alpha` tag, although they have been tested thoroughly several times \r\n- No change is required in the way `Metrics` or any jobs are configured \r\n- More documentation will follow in Wiki PR \r\n","created_at":"2016-07-06T07:43:50Z","updated_at":"2016-07-12T22:33:09Z","closed_at":null,"merged_at":null,"merge_commit_sha":"a7a945e67cf9d96645e7f32e0334cd91c93668b3","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/linkedin/gobblin/pulls/1088/commits","review_comments_url":"https://api.github.com/repos/linkedin/gobblin/pulls/1088/comments","review_comment_url":"https://api.github.com/repos/linkedin/gobblin/pulls/comments{/number}","comments_url":"https://api.github.com/repos/linkedin/gobblin/issues/1088/comments","statuses_url":"https://api.github.com/repos/linkedin/gobblin/statuses/2216b005d801f9c69bb50e4535079872966554b3","head":{"label":"abti:aws","ref":"aws","sha":"2216b005d801f9c69bb50e4535079872966554b3","user":{"login":"abti","id":69499,"avatar_url":"https://avatars.githubusercontent.com/u/69499?v=3","gravatar_id":"","url":"https://api.github.com/users/abti","html_url":"https://github.com/abti","followers_url":"https://api.github.com/users/abti/followers","following_url":"https://api.github.com/users/abti/following{/other_user}","gists_url":"https://api.github.com/users/abti/gists{/gist_id}","starred_url":"https://api.github.com/users/abti/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/abti/subscriptions","organizations_url":"https://api.github.com/users/abti/orgs","repos_url":"https://api.github.com/users/abti/repos","events_url":"https://api.github.com/users/abti/events{/privacy}","received_events_url":"https://api.github.com/users/abti/received_events","type":"User","site_admin":false},"repo":{"id":42212016,"name":"gobblin","full_name":"abti/gobblin","owner":{"login":"abti","id":69499,"avatar_url":"https://avatars.githubusercontent.com/u/69499?v=3","gravatar_id":"","url":"https://api.github.com/users/abti","html_url":"https://github.com/abti","followers_url":"https://api.github.com/users/abti/followers","following_url":"https://api.github.com/users/abti/following{/other_user}","gists_url":"https://api.github.com/users/abti/gists{/gist_id}","starred_url":"https://api.github.com/users/abti/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/abti/subscriptions","organizations_url":"https://api.github.com/users/abti/orgs","repos_url":"https://api.github.com/users/abti/repos","events_url":"https://api.github.com/users/abti/events{/privacy}","received_events_url":"https://api.github.com/users/abti/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/abti/gobblin","description":"Universal data ingestion framework for Hadoop.","fork":true,"url":"https://api.github.com/repos/abti/gobblin","forks_url":"https://api.github.com/repos/abti/gobblin/forks","keys_url":"https://api.github.com/repos/abti/gobblin/keys{/key_id}","collaborators_url":"https://api.github.com/repos/abti/gobblin/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/abti/gobblin/teams","hooks_url":"https://api.github.com/repos/abti/gobblin/hooks","issue_events_url":"https://api.github.com/repos/abti/gobblin/issues/events{/number}","events_url":"https://api.github.com/repos/abti/gobblin/events","assignees_url":"https://api.github.com/repos/abti/gobblin/assignees{/user}","branches_url":"https://api.github.com/repos/abti/gobblin/branches{/branch}","tags_url":"https://api.github.com/repos/abti/gobblin/tags","blobs_url":"https://api.github.com/repos/abti/gobblin/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/abti/gobblin/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/abti/gobblin/git/refs{/sha}","trees_url":"https://api.github.com/repos/abti/gobblin/git/trees{/sha}","statuses_url":"https://api.github.com/repos/abti/gobblin/statuses/{sha}","languages_url":"https://api.github.com/repos/abti/gobblin/languages","stargazers_url":"https://api.github.com/repos/abti/gobblin/stargazers","contributors_url":"https://api.github.com/repos/abti/gobblin/contributors","subscribers_url":"https://api.github.com/repos/abti/gobblin/subscribers","subscription_url":"https://api.github.com/repos/abti/gobblin/subscription","commits_url":"https://api.github.com/repos/abti/gobblin/commits{/sha}","git_commits_url":"https://api.github.com/repos/abti/gobblin/git/commits{/sha}","comments_url":"https://api.github.com/repos/abti/gobblin/comments{/number}","issue_comment_url":"https://api.github.com/repos/abti/gobblin/issues/comments{/number}","contents_url":"https://api.github.com/repos/abti/gobblin/contents/{+path}","compare_url":"https://api.github.com/repos/abti/gobblin/compare/{base}...{head}","merges_url":"https://api.github.com/repos/abti/gobblin/merges","archive_url":"https://api.github.com/repos/abti/gobblin/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/abti/gobblin/downloads","issues_url":"https://api.github.com/repos/abti/gobblin/issues{/number}","pulls_url":"https://api.github.com/repos/abti/gobblin/pulls{/number}","milestones_url":"https://api.github.com/repos/abti/gobblin/milestones{/number}","notifications_url":"https://api.github.com/repos/abti/gobblin/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/abti/gobblin/labels{/name}","releases_url":"https://api.github.com/repos/abti/gobblin/releases{/id}","deployments_url":"https://api.github.com/repos/abti/gobblin/deployments","created_at":"2015-09-10T00:25:20Z","updated_at":"2016-01-14T21:11:37Z","pushed_at":"2016-07-12T22:19:31Z","git_url":"git://github.com/abti/gobblin.git","ssh_url":"git@github.com:abti/gobblin.git","clone_url":"https://github.com/abti/gobblin.git","svn_url":"https://github.com/abti/gobblin","homepage":"","size":87387,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"linkedin:master","ref":"master","sha":"589c1bc4447bc59e9cb4c87ecc1b6b72a288515f","user":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"repo":{"id":27392543,"name":"gobblin","full_name":"linkedin/gobblin","owner":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/linkedin/gobblin","description":"Universal data ingestion framework for Hadoop.","fork":false,"url":"https://api.github.com/repos/linkedin/gobblin","forks_url":"https://api.github.com/repos/linkedin/gobblin/forks","keys_url":"https://api.github.com/repos/linkedin/gobblin/keys{/key_id}","collaborators_url":"https://api.github.com/repos/linkedin/gobblin/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/linkedin/gobblin/teams","hooks_url":"https://api.github.com/repos/linkedin/gobblin/hooks","issue_events_url":"https://api.github.com/repos/linkedin/gobblin/issues/events{/number}","events_url":"https://api.github.com/repos/linkedin/gobblin/events","assignees_url":"https://api.github.com/repos/linkedin/gobblin/assignees{/user}","branches_url":"https://api.github.com/repos/linkedin/gobblin/branches{/branch}","tags_url":"https://api.github.com/repos/linkedin/gobblin/tags","blobs_url":"https://api.github.com/repos/linkedin/gobblin/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/linkedin/gobblin/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/linkedin/gobblin/git/refs{/sha}","trees_url":"https://api.github.com/repos/linkedin/gobblin/git/trees{/sha}","statuses_url":"https://api.github.com/repos/linkedin/gobblin/statuses/{sha}","languages_url":"https://api.github.com/repos/linkedin/gobblin/languages","stargazers_url":"https://api.github.com/repos/linkedin/gobblin/stargazers","contributors_url":"https://api.github.com/repos/linkedin/gobblin/contributors","subscribers_url":"https://api.github.com/repos/linkedin/gobblin/subscribers","subscription_url":"https://api.github.com/repos/linkedin/gobblin/subscription","commits_url":"https://api.github.com/repos/linkedin/gobblin/commits{/sha}","git_commits_url":"https://api.github.com/repos/linkedin/gobblin/git/commits{/sha}","comments_url":"https://api.github.com/repos/linkedin/gobblin/comments{/number}","issue_comment_url":"https://api.github.com/repos/linkedin/gobblin/issues/comments{/number}","contents_url":"https://api.github.com/repos/linkedin/gobblin/contents/{+path}","compare_url":"https://api.github.com/repos/linkedin/gobblin/compare/{base}...{head}","merges_url":"https://api.github.com/repos/linkedin/gobblin/merges","archive_url":"https://api.github.com/repos/linkedin/gobblin/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/linkedin/gobblin/downloads","issues_url":"https://api.github.com/repos/linkedin/gobblin/issues{/number}","pulls_url":"https://api.github.com/repos/linkedin/gobblin/pulls{/number}","milestones_url":"https://api.github.com/repos/linkedin/gobblin/milestones{/number}","notifications_url":"https://api.github.com/repos/linkedin/gobblin/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/linkedin/gobblin/labels{/name}","releases_url":"https://api.github.com/repos/linkedin/gobblin/releases{/id}","deployments_url":"https://api.github.com/repos/linkedin/gobblin/deployments","created_at":"2014-12-01T18:10:50Z","updated_at":"2016-07-11T12:56:12Z","pushed_at":"2016-07-12T22:19:32Z","git_url":"git://github.com/linkedin/gobblin.git","ssh_url":"git@github.com:linkedin/gobblin.git","clone_url":"https://github.com/linkedin/gobblin.git","svn_url":"https://github.com/linkedin/gobblin","homepage":"https://github.com/linkedin/gobblin/wiki","size":172913,"stargazers_count":633,"watchers_count":633,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":259,"mirror_url":null,"open_issues_count":109,"forks":259,"open_issues":109,"watchers":633,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/1088"},"html":{"href":"https://github.com/linkedin/gobblin/pull/1088"},"issue":{"href":"https://api.github.com/repos/linkedin/gobblin/issues/1088"},"comments":{"href":"https://api.github.com/repos/linkedin/gobblin/issues/1088/comments"},"review_comments":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/1088/comments"},"review_comment":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/1088/commits"},"statuses":{"href":"https://api.github.com/repos/linkedin/gobblin/statuses/2216b005d801f9c69bb50e4535079872966554b3"}}}}
{"actor":{"display_login":"abti"}}