{"action":"created","comment":{"url":"https://api.github.com/repos/stubhub/bigdime/pulls/comments/61176826","id":61176826,"diff_hunk":"@@ -0,0 +1,530 @@\n+/**\r\n+ * Copyright (C) 2015 Stubhub.\r\n+ */\r\n+package io.bigdime.handler.file;\r\n+\r\n+import io.bigdime.alert.Logger.ALERT_CAUSE;\r\n+import io.bigdime.alert.Logger.ALERT_SEVERITY;\r\n+import io.bigdime.alert.Logger.ALERT_TYPE;\r\n+import io.bigdime.alert.LoggerFactory;\r\n+import io.bigdime.core.ActionEvent;\r\n+import io.bigdime.core.ActionEvent.Status;\r\n+import io.bigdime.core.AdaptorConfigurationException;\r\n+import io.bigdime.core.HandlerException;\r\n+import io.bigdime.core.InvalidValueConfigurationException;\r\n+import io.bigdime.core.commons.AdaptorLogger;\r\n+import io.bigdime.core.commons.DataConstants;\r\n+import io.bigdime.core.commons.FileHelper;\r\n+import io.bigdime.core.commons.PropertyHelper;\r\n+import io.bigdime.core.config.AdaptorConfigConstants;\r\n+import io.bigdime.core.constants.ActionEventHeaderConstants;\r\n+import io.bigdime.core.handler.AbstractHandler;\r\n+import io.bigdime.core.handler.HandlerJournal;\r\n+import io.bigdime.core.runtimeinfo.RuntimeInfo;\r\n+import io.bigdime.core.runtimeinfo.RuntimeInfoStore;\r\n+import io.bigdime.core.runtimeinfo.RuntimeInfoStoreException;\r\n+\r\n+import java.io.BufferedInputStream;\r\n+import java.io.File;\r\n+import java.io.IOException;\r\n+import java.io.InputStream;\r\n+import java.util.Enumeration;\r\n+import java.util.List;\r\n+import java.util.Map.Entry;\r\n+import java.util.zip.ZipEntry;\r\n+import java.util.zip.ZipException;\r\n+import java.util.zip.ZipFile;\r\n+\r\n+import org.apache.commons.lang.NotImplementedException;\r\n+import org.springframework.beans.factory.annotation.Autowired;\r\n+import org.springframework.context.annotation.Scope;\r\n+import org.springframework.stereotype.Component;\r\n+\r\n+\r\n+/**\r\n+ * A handler that reads data from a file. The data is read in binary or text\r\n+ * depending on the handler configuration.\r\n+ * @formatter:off\r\n+ * Read a file from given location.\r\n+ * Read the location of the file.\r\n+ * Read the buffer size.\r\n+ * if context is null\r\n+ * \tstart_index=0;\r\n+ * else\r\n+ * \tstart_index=bytes_read_already\r\n+ *\r\n+ *\r\n+ * File:\r\n+ *\r\n+ * Use multiple channels if:\r\n+ * reader is faster\r\n+ * record boundary\r\n+ * order of records doesn't matter.\r\n+ *\r\n+ * Use single outputChannel if any of the below is true:\r\n+ * checksum is required\r\n+ * order or the records need to be maintained\r\n+ * @formatter:on\r\n+ * @author Neeraj Jain, Rita Liu\r\n+ *\r\n+ */\r\n+@Component\r\n+@Scope(\"prototype\")\r\n+public class ZipFileInputStreamHandler extends AbstractHandler {\r\n+\tprivate static final AdaptorLogger logger = new AdaptorLogger(\r\n+\t\t\tLoggerFactory.getLogger(ZipFileInputStreamHandler.class));\r\n+\tpublic static final String FILE_LOCATION = \"fileLocation\";\r\n+\r\n+\tprivate static int DEFAULT_BUFFER_SIZE = 1024 * 1024;\r\n+\r\n+\tprivate File currentFile;\r\n+\r\n+\tprivate String fileNamePattern;\r\n+\r\n+\tprivate int bufferSize;\r\n+\r\n+\t/**\r\n+\t * Base path, absolute, to look for the files.\r\n+\t */\r\n+\tprivate String fileLocation;\r\n+\r\n+\tprivate long zipFileLength = -1;\r\n+\r\n+\tprivate FileInputDescriptor fileInputDescriptor;\r\n+\r\n+\t@Autowired\r\n+\tprivate RuntimeInfoStore<RuntimeInfo> runtimeInfoStore;\r\n+\t@Autowired\r\n+\tprivate FileHelper fileHelper;\r\n+\r\n+\tprivate String handlerPhase = \"\";\r\n+\tprivate String entityName;\r\n+\r\n+\tprivate File currentFileToProcess;\r\n+\tprivate int readEntries = 0;\r\n+\tprivate String basePath = \"/\";\r\n+\tprivate boolean preserveBasePath;\r\n+\tprivate boolean preserveRelativePath;\r\n+\tprivate ZipEntry zipFileEntry;\r\n+\tprivate ZipFile zipFile;\r\n+\tprivate InputStream inputStream;\r\n+\tprivate BufferedInputStream bufferedInputStream;\r\n+\tprivate int noOfEntry;\r\n+\tprivate String fileName = \"\";\r\n+\tprivate long totalRead = 0;\r\n+\tprivate long fileSize = 0;\r\n+\tEnumeration<? extends ZipEntry> entriesInZip;\r\n+\t\t\r\n+\t@Override\r\n+\tpublic void build() throws AdaptorConfigurationException {\r\n+\t\tsuper.build();\r\n+\t\thandlerPhase = \"building ZipFileInputStreamHandler\";\r\n+\t\tlogger.info(handlerPhase, \"properties={}\", getPropertyMap());\r\n+\r\n+\t\t@SuppressWarnings(\"unchecked\")\r\n+\t\tEntry<String, String> srcDescInputs = (Entry<String, String>) getPropertyMap()\r\n+\t\t\t\t.get(AdaptorConfigConstants.SourceConfigConstants.SRC_DESC);\r\n+\t\tif (srcDescInputs == null) {\r\n+\t\t\tthrow new InvalidValueConfigurationException(\"src-desc can't be null\");\r\n+\t\t}\r\n+\t\tlogger.debug(handlerPhase, \"entity:fileNamePattern={} input_field_name={}\", srcDescInputs.getKey(),\r\n+\t\t\t\tsrcDescInputs.getValue());\r\n+\r\n+\t\tbasePath = PropertyHelper.getStringProperty(getPropertyMap(), FileInputStreamReaderHandlerConstants.BASE_PATH,\r\n+\t\t\t\tDataConstants.SLASH);\r\n+\t\tlogger.debug(handlerPhase, \"basePath={}\", basePath);\r\n+\r\n+\t\tfileInputDescriptor = new FileInputDescriptor();\r\n+\t\ttry {\r\n+\t\t\tfileInputDescriptor.parseDescriptor(basePath);\r\n+\t\t} catch (IllegalArgumentException ex) {\r\n+\t\t\tthrow new InvalidValueConfigurationException(\r\n+\t\t\t\t\t\"src-desc must contain entityName:fileNamePrefix, e.g. user:web_user*\");\r\n+\t\t}\r\n+\r\n+\t\tfileLocation = fileInputDescriptor.getPath();\r\n+\t\tif (fileLocation == null) {\r\n+\t\t\tthrow new InvalidValueConfigurationException(\"filePath in src-desc can't be null\");\r\n+\t\t}\r\n+\r\n+\t\tString[] srcDesc = fileInputDescriptor.parseSourceDescriptor(srcDescInputs.getKey());\r\n+\r\n+\t\tentityName = srcDesc[0];\r\n+\t\tfileNamePattern = srcDesc[1];\r\n+\t\tlogger.debug(handlerPhase, \"entityName={} fileNamePattern={}\", entityName, fileNamePattern);\r\n+\r\n+\t\tbufferSize = PropertyHelper.getIntProperty(getPropertyMap(), FileInputStreamReaderHandlerConstants.BUFFER_SIZE,\r\n+\t\t\t\tDEFAULT_BUFFER_SIZE);\r\n+\t\tlogger.debug(handlerPhase, \"id={} fileLocation={}\", getId(), fileLocation);\r\n+\t\tpreserveBasePath = PropertyHelper.getBooleanProperty(getPropertyMap(),\r\n+\t\t\t\tFileInputStreamReaderHandlerConstants.PRESERVE_BASE_PATH);\r\n+\t\tpreserveRelativePath = PropertyHelper.getBooleanProperty(getPropertyMap(),\r\n+\t\t\t\tFileInputStreamReaderHandlerConstants.PRESERVE_RELATIVE_PATH);\r\n+\r\n+\t\tlogger.debug(handlerPhase, \"id={}\", getId());\r\n+\t}\r\n+\r\n+\t@Override\r\n+\tpublic Status process() throws HandlerException {\r\n+\t\thandlerPhase = \"processing ZipFileInputStreamHandler\";\r\n+\t\tincrementInvocationCount();\r\n+\t\ttry {\r\n+\t\t\tStatus status = preProcess();\r\n+\t\t\tif (status == Status.BACKOFF) {\r\n+\t\t\t\tlogger.debug(handlerPhase, \"returning BACKOFF\");\r\n+\t\t\t\treturn status;\r\n+\t\t\t}\r\n+\t\t\treturn doProcess();\r\n+\t\t} catch (IOException e) {\r\n+\t\t\tlogger.alert(ALERT_TYPE.INGESTION_FAILED, ALERT_CAUSE.APPLICATION_INTERNAL_ERROR, ALERT_SEVERITY.BLOCKER,\r\n+\t\t\t\t\t\"error during reading file\", e);\r\n+\t\t\tthrow new HandlerException(\"Unable to process message from file\", e);\r\n+\t\t} catch (RuntimeInfoStoreException e) {\r\n+\t\t\tthrow new HandlerException(\"Unable to process message from file\", e);\r\n+\t\t}\r\n+\t}\r\n+\t\r\n+\tprivate boolean isFirstRun() {\r\n+\t\treturn getInvocationCount() == 1;\r\n+\t}\r\n+\t\r\n+\tlong dirtyRecordCount = 0;\r\n+\tList<RuntimeInfo> dirtyRecords;\r\n+\tprivate boolean processingDirty = false;\r\n+\t\r\n+\tprivate void getStartedRecordsFromRuntimeInfos() throws RuntimeInfoStoreException{\r\n+\t\tdirtyRecords = getAllStartedRuntimeInfos(runtimeInfoStore, entityName);\r\n+\t\tif(dirtyRecords != null && !dirtyRecords.isEmpty()){\r\n+\t\t\tdirtyRecordCount = dirtyRecords.size();\r\n+\t\t\tlogger.warn(handlerPhase,\r\n+\t\t\t\t\t\"_message=\\\"dirty records found\\\" handler_id={} dirty_record_count=\\\"{}\\\" entityName={}\",\r\n+\t\t\t\t\tgetId(), dirtyRecordCount, entityName);\r\n+\t\t} else {\r\n+\t\t\tlogger.info(handlerPhase, \"_message=\\\"no dirty records found\\\" handler_id={}\", getId());\r\n+\t\t}\r\n+\t}\r\n+\r\n+\tprivate Status preProcess() throws IOException, RuntimeInfoStoreException, HandlerException {\r\n+\t\t\r\n+\t\tif(isFirstRun()){\r\n+\t\t\tgetStartedRecordsFromRuntimeInfos();\r\n+\t\t}\r\n+\t\t\r\n+\t\tif (readAllFromZip() && readAllFromFile()) {\r\n+\t\t\tcurrentFileToProcess = getNextFileToProcess();\r\n+\t\t\tif (currentFileToProcess == null) {\t\t\t\r\n+\t\t\t\tlogger.info(handlerPhase, \"_message=\\\"no file to process\\\" handler_id={} \", getId());\r\n+\t\t\t\treturn Status.BACKOFF;\r\n+\t\t\t}\r\n+\t\t\tnoOfEntry = getNumberOfFileInZip(currentFileToProcess);\r\n+\t\t\treadEntries++;\r\n+\t\t\tzipFileLength = currentFileToProcess.length();\r\n+\t\t\tlogger.info(handlerPhase, \"_message=\\\"got a new file to process\\\" handler_id={} zip_file_length={}\", getId(),\r\n+\t\t\t\t\tzipFileLength);\r\n+\t\t\tif (zipFileLength == 0) {\r\n+\t\t\t\tlogger.info(handlerPhase, \"_message=\\\"zip file is empty\\\" handler_id={} \", getId());\r\n+\t\t\t\treturn Status.BACKOFF;\r\n+\t\t\t}\r\n+\t\t\tlogger.info(handlerPhase, \"_message=\\\"number files in zip\\\" handler_id={} no_of_file={} \", getId(),noOfEntry);\r\n+\t\t\tgetZipFileHandlerJournal().setTotalEntries(noOfEntry);\r\n+\t\t\tgetZipFileHandlerJournal().setZipFileName(currentFileToProcess.getName());\r\n+\t\t\tgetZipFileHandlerJournal().setReadEntries(readEntries);\r\n+\t\t\treturn Status.READY;\r\n+\t\t}\r\n+\t\treturn Status.READY;\r\n+\t}\r\n+\r\n+\tprivate Status doProcess() throws IOException, HandlerException, RuntimeInfoStoreException {\r\n+\t\t\r\n+\t\tbyte[] data ;\r\n+\t\tlong bytesReadCount=0;\r\n+\t\tint readSize =0;\r\n+\t\tString srcFileName = \"\";\r\n+\t\t\r\n+\t\tlogger.debug(handlerPhase, \"handler_id={} next_index_to_read={} buffer_size={}\", getId(), getTotalReadFromJournal(),\r\n+\t\t\t\tbufferSize);\r\n+\t\tif(getReadEntriesFromJournal() == 1 && readAllFromFile()){\r\n+\t\t\tentriesInZip = zipFile.entries();\r","path":"bigdime-data-handlers/src/main/java/io/bigdime/handler/file/ZipFileInputStreamHandler.java","position":247,"original_position":247,"commit_id":"9755d80e7313a34cb1b9ec8de84c84f34b764da8","original_commit_id":"9755d80e7313a34cb1b9ec8de84c84f34b764da8","user":{"login":"a-ninja","id":5307299,"avatar_url":"https://avatars.githubusercontent.com/u/5307299?v=3","gravatar_id":"","url":"https://api.github.com/users/a-ninja","html_url":"https://github.com/a-ninja","followers_url":"https://api.github.com/users/a-ninja/followers","following_url":"https://api.github.com/users/a-ninja/following{/other_user}","gists_url":"https://api.github.com/users/a-ninja/gists{/gist_id}","starred_url":"https://api.github.com/users/a-ninja/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/a-ninja/subscriptions","organizations_url":"https://api.github.com/users/a-ninja/orgs","repos_url":"https://api.github.com/users/a-ninja/repos","events_url":"https://api.github.com/users/a-ninja/events{/privacy}","received_events_url":"https://api.github.com/users/a-ninja/received_events","type":"User","site_admin":false},"body":"Why not move it to preProcess?","created_at":"2016-04-26T22:37:36Z","updated_at":"2016-04-26T22:37:36Z","html_url":"https://github.com/stubhub/bigdime/pull/47#discussion_r61176826","pull_request_url":"https://api.github.com/repos/stubhub/bigdime/pulls/47","_links":{"self":{"href":"https://api.github.com/repos/stubhub/bigdime/pulls/comments/61176826"},"html":{"href":"https://github.com/stubhub/bigdime/pull/47#discussion_r61176826"},"pull_request":{"href":"https://api.github.com/repos/stubhub/bigdime/pulls/47"}}},"pull_request":{"url":"https://api.github.com/repos/stubhub/bigdime/pulls/47","id":66226048,"html_url":"https://github.com/stubhub/bigdime/pull/47","diff_url":"https://github.com/stubhub/bigdime/pull/47.diff","patch_url":"https://github.com/stubhub/bigdime/pull/47.patch","issue_url":"https://api.github.com/repos/stubhub/bigdime/issues/47","number":47,"state":"open","locked":false,"title":"check in zip file handler and validation","user":{"login":"ritaliu912","id":16546388,"avatar_url":"https://avatars.githubusercontent.com/u/16546388?v=3","gravatar_id":"","url":"https://api.github.com/users/ritaliu912","html_url":"https://github.com/ritaliu912","followers_url":"https://api.github.com/users/ritaliu912/followers","following_url":"https://api.github.com/users/ritaliu912/following{/other_user}","gists_url":"https://api.github.com/users/ritaliu912/gists{/gist_id}","starred_url":"https://api.github.com/users/ritaliu912/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ritaliu912/subscriptions","organizations_url":"https://api.github.com/users/ritaliu912/orgs","repos_url":"https://api.github.com/users/ritaliu912/repos","events_url":"https://api.github.com/users/ritaliu912/events{/privacy}","received_events_url":"https://api.github.com/users/ritaliu912/received_events","type":"User","site_admin":false},"body":"","created_at":"2016-04-12T20:52:43Z","updated_at":"2016-04-26T22:37:36Z","closed_at":null,"merged_at":null,"merge_commit_sha":"3bcaf059a45c8c2d59e2a1c62ef44765f62e996f","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/stubhub/bigdime/pulls/47/commits","review_comments_url":"https://api.github.com/repos/stubhub/bigdime/pulls/47/comments","review_comment_url":"https://api.github.com/repos/stubhub/bigdime/pulls/comments{/number}","comments_url":"https://api.github.com/repos/stubhub/bigdime/issues/47/comments","statuses_url":"https://api.github.com/repos/stubhub/bigdime/statuses/9755d80e7313a34cb1b9ec8de84c84f34b764da8","head":{"label":"ritaliu912:develop","ref":"develop","sha":"9755d80e7313a34cb1b9ec8de84c84f34b764da8","user":{"login":"ritaliu912","id":16546388,"avatar_url":"https://avatars.githubusercontent.com/u/16546388?v=3","gravatar_id":"","url":"https://api.github.com/users/ritaliu912","html_url":"https://github.com/ritaliu912","followers_url":"https://api.github.com/users/ritaliu912/followers","following_url":"https://api.github.com/users/ritaliu912/following{/other_user}","gists_url":"https://api.github.com/users/ritaliu912/gists{/gist_id}","starred_url":"https://api.github.com/users/ritaliu912/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ritaliu912/subscriptions","organizations_url":"https://api.github.com/users/ritaliu912/orgs","repos_url":"https://api.github.com/users/ritaliu912/repos","events_url":"https://api.github.com/users/ritaliu912/events{/privacy}","received_events_url":"https://api.github.com/users/ritaliu912/received_events","type":"User","site_admin":false},"repo":{"id":53965947,"name":"bigdime-1","full_name":"ritaliu912/bigdime-1","owner":{"login":"ritaliu912","id":16546388,"avatar_url":"https://avatars.githubusercontent.com/u/16546388?v=3","gravatar_id":"","url":"https://api.github.com/users/ritaliu912","html_url":"https://github.com/ritaliu912","followers_url":"https://api.github.com/users/ritaliu912/followers","following_url":"https://api.github.com/users/ritaliu912/following{/other_user}","gists_url":"https://api.github.com/users/ritaliu912/gists{/gist_id}","starred_url":"https://api.github.com/users/ritaliu912/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ritaliu912/subscriptions","organizations_url":"https://api.github.com/users/ritaliu912/orgs","repos_url":"https://api.github.com/users/ritaliu912/repos","events_url":"https://api.github.com/users/ritaliu912/events{/privacy}","received_events_url":"https://api.github.com/users/ritaliu912/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/ritaliu912/bigdime-1","description":"As a Big Data Ingestion Ecosystem, BigDime enables all types and formats of high volume data into the Hadoop Big Data environment. Loaded with several features and capabilities right out of the box, such as Data Ingestion, Monitoring, Alerting, Metadata Management, Data Validation Service and High Availability,  BigDime aspires to close all functional gaps discovered throughout Big Data Ingestion challenges that most data ambitious organizations experience.","fork":true,"url":"https://api.github.com/repos/ritaliu912/bigdime-1","forks_url":"https://api.github.com/repos/ritaliu912/bigdime-1/forks","keys_url":"https://api.github.com/repos/ritaliu912/bigdime-1/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ritaliu912/bigdime-1/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ritaliu912/bigdime-1/teams","hooks_url":"https://api.github.com/repos/ritaliu912/bigdime-1/hooks","issue_events_url":"https://api.github.com/repos/ritaliu912/bigdime-1/issues/events{/number}","events_url":"https://api.github.com/repos/ritaliu912/bigdime-1/events","assignees_url":"https://api.github.com/repos/ritaliu912/bigdime-1/assignees{/user}","branches_url":"https://api.github.com/repos/ritaliu912/bigdime-1/branches{/branch}","tags_url":"https://api.github.com/repos/ritaliu912/bigdime-1/tags","blobs_url":"https://api.github.com/repos/ritaliu912/bigdime-1/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ritaliu912/bigdime-1/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ritaliu912/bigdime-1/git/refs{/sha}","trees_url":"https://api.github.com/repos/ritaliu912/bigdime-1/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ritaliu912/bigdime-1/statuses/{sha}","languages_url":"https://api.github.com/repos/ritaliu912/bigdime-1/languages","stargazers_url":"https://api.github.com/repos/ritaliu912/bigdime-1/stargazers","contributors_url":"https://api.github.com/repos/ritaliu912/bigdime-1/contributors","subscribers_url":"https://api.github.com/repos/ritaliu912/bigdime-1/subscribers","subscription_url":"https://api.github.com/repos/ritaliu912/bigdime-1/subscription","commits_url":"https://api.github.com/repos/ritaliu912/bigdime-1/commits{/sha}","git_commits_url":"https://api.github.com/repos/ritaliu912/bigdime-1/git/commits{/sha}","comments_url":"https://api.github.com/repos/ritaliu912/bigdime-1/comments{/number}","issue_comment_url":"https://api.github.com/repos/ritaliu912/bigdime-1/issues/comments{/number}","contents_url":"https://api.github.com/repos/ritaliu912/bigdime-1/contents/{+path}","compare_url":"https://api.github.com/repos/ritaliu912/bigdime-1/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ritaliu912/bigdime-1/merges","archive_url":"https://api.github.com/repos/ritaliu912/bigdime-1/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ritaliu912/bigdime-1/downloads","issues_url":"https://api.github.com/repos/ritaliu912/bigdime-1/issues{/number}","pulls_url":"https://api.github.com/repos/ritaliu912/bigdime-1/pulls{/number}","milestones_url":"https://api.github.com/repos/ritaliu912/bigdime-1/milestones{/number}","notifications_url":"https://api.github.com/repos/ritaliu912/bigdime-1/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ritaliu912/bigdime-1/labels{/name}","releases_url":"https://api.github.com/repos/ritaliu912/bigdime-1/releases{/id}","deployments_url":"https://api.github.com/repos/ritaliu912/bigdime-1/deployments","created_at":"2016-03-15T17:46:11Z","updated_at":"2016-03-15T17:46:16Z","pushed_at":"2016-04-13T18:10:55Z","git_url":"git://github.com/ritaliu912/bigdime-1.git","ssh_url":"git@github.com:ritaliu912/bigdime-1.git","clone_url":"https://github.com/ritaliu912/bigdime-1.git","svn_url":"https://github.com/ritaliu912/bigdime-1","homepage":"http://www.bigdime.io ","size":5688,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"develop"}},"base":{"label":"stubhub:develop","ref":"develop","sha":"33907dee289a2af1ef72ce6a62a3c45267817eed","user":{"login":"stubhub","id":8224322,"avatar_url":"https://avatars.githubusercontent.com/u/8224322?v=3","gravatar_id":"","url":"https://api.github.com/users/stubhub","html_url":"https://github.com/stubhub","followers_url":"https://api.github.com/users/stubhub/followers","following_url":"https://api.github.com/users/stubhub/following{/other_user}","gists_url":"https://api.github.com/users/stubhub/gists{/gist_id}","starred_url":"https://api.github.com/users/stubhub/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/stubhub/subscriptions","organizations_url":"https://api.github.com/users/stubhub/orgs","repos_url":"https://api.github.com/users/stubhub/repos","events_url":"https://api.github.com/users/stubhub/events{/privacy}","received_events_url":"https://api.github.com/users/stubhub/received_events","type":"Organization","site_admin":false},"repo":{"id":48773470,"name":"bigdime","full_name":"stubhub/bigdime","owner":{"login":"stubhub","id":8224322,"avatar_url":"https://avatars.githubusercontent.com/u/8224322?v=3","gravatar_id":"","url":"https://api.github.com/users/stubhub","html_url":"https://github.com/stubhub","followers_url":"https://api.github.com/users/stubhub/followers","following_url":"https://api.github.com/users/stubhub/following{/other_user}","gists_url":"https://api.github.com/users/stubhub/gists{/gist_id}","starred_url":"https://api.github.com/users/stubhub/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/stubhub/subscriptions","organizations_url":"https://api.github.com/users/stubhub/orgs","repos_url":"https://api.github.com/users/stubhub/repos","events_url":"https://api.github.com/users/stubhub/events{/privacy}","received_events_url":"https://api.github.com/users/stubhub/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/stubhub/bigdime","description":"As a Big Data Ingestion Ecosystem, BigDime enables all types and formats of high volume data into the Hadoop Big Data environment. Loaded with several features and capabilities right out of the box, such as Data Ingestion, Monitoring, Alerting, Metadata Management, Data Validation Service and High Availability,  BigDime aspires to close all functional gaps discovered throughout Big Data Ingestion challenges that most data ambitious organizations experience.","fork":false,"url":"https://api.github.com/repos/stubhub/bigdime","forks_url":"https://api.github.com/repos/stubhub/bigdime/forks","keys_url":"https://api.github.com/repos/stubhub/bigdime/keys{/key_id}","collaborators_url":"https://api.github.com/repos/stubhub/bigdime/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/stubhub/bigdime/teams","hooks_url":"https://api.github.com/repos/stubhub/bigdime/hooks","issue_events_url":"https://api.github.com/repos/stubhub/bigdime/issues/events{/number}","events_url":"https://api.github.com/repos/stubhub/bigdime/events","assignees_url":"https://api.github.com/repos/stubhub/bigdime/assignees{/user}","branches_url":"https://api.github.com/repos/stubhub/bigdime/branches{/branch}","tags_url":"https://api.github.com/repos/stubhub/bigdime/tags","blobs_url":"https://api.github.com/repos/stubhub/bigdime/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/stubhub/bigdime/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/stubhub/bigdime/git/refs{/sha}","trees_url":"https://api.github.com/repos/stubhub/bigdime/git/trees{/sha}","statuses_url":"https://api.github.com/repos/stubhub/bigdime/statuses/{sha}","languages_url":"https://api.github.com/repos/stubhub/bigdime/languages","stargazers_url":"https://api.github.com/repos/stubhub/bigdime/stargazers","contributors_url":"https://api.github.com/repos/stubhub/bigdime/contributors","subscribers_url":"https://api.github.com/repos/stubhub/bigdime/subscribers","subscription_url":"https://api.github.com/repos/stubhub/bigdime/subscription","commits_url":"https://api.github.com/repos/stubhub/bigdime/commits{/sha}","git_commits_url":"https://api.github.com/repos/stubhub/bigdime/git/commits{/sha}","comments_url":"https://api.github.com/repos/stubhub/bigdime/comments{/number}","issue_comment_url":"https://api.github.com/repos/stubhub/bigdime/issues/comments{/number}","contents_url":"https://api.github.com/repos/stubhub/bigdime/contents/{+path}","compare_url":"https://api.github.com/repos/stubhub/bigdime/compare/{base}...{head}","merges_url":"https://api.github.com/repos/stubhub/bigdime/merges","archive_url":"https://api.github.com/repos/stubhub/bigdime/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/stubhub/bigdime/downloads","issues_url":"https://api.github.com/repos/stubhub/bigdime/issues{/number}","pulls_url":"https://api.github.com/repos/stubhub/bigdime/pulls{/number}","milestones_url":"https://api.github.com/repos/stubhub/bigdime/milestones{/number}","notifications_url":"https://api.github.com/repos/stubhub/bigdime/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/stubhub/bigdime/labels{/name}","releases_url":"https://api.github.com/repos/stubhub/bigdime/releases{/id}","deployments_url":"https://api.github.com/repos/stubhub/bigdime/deployments","created_at":"2015-12-30T00:41:00Z","updated_at":"2016-04-12T18:18:11Z","pushed_at":"2016-04-25T06:38:44Z","git_url":"git://github.com/stubhub/bigdime.git","ssh_url":"git@github.com:stubhub/bigdime.git","clone_url":"https://github.com/stubhub/bigdime.git","svn_url":"https://github.com/stubhub/bigdime","homepage":"http://www.bigdime.io ","size":5756,"stargazers_count":2,"watchers_count":2,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":8,"mirror_url":null,"open_issues_count":7,"forks":8,"open_issues":7,"watchers":2,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/stubhub/bigdime/pulls/47"},"html":{"href":"https://github.com/stubhub/bigdime/pull/47"},"issue":{"href":"https://api.github.com/repos/stubhub/bigdime/issues/47"},"comments":{"href":"https://api.github.com/repos/stubhub/bigdime/issues/47/comments"},"review_comments":{"href":"https://api.github.com/repos/stubhub/bigdime/pulls/47/comments"},"review_comment":{"href":"https://api.github.com/repos/stubhub/bigdime/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/stubhub/bigdime/pulls/47/commits"},"statuses":{"href":"https://api.github.com/repos/stubhub/bigdime/statuses/9755d80e7313a34cb1b9ec8de84c84f34b764da8"}}}}