sparqlGraphIngestionService/src/main/java/com/ge/research/semtk/services/ingestion/IngestionRestController.java
private JSONObject fromAnyCsv(Object templateFile, Object dataFile, Object sparqlConnectionOverride, Boolean fromFiles, Boolean precheck, Boolean skipIngest, Boolean trackFlag, String overrideBaseURI){
		
		DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");

		int recordsProcessed = 0;
		
		// set up the logger
		LoggerRestClient logger = null;
		logger = getLoggerRestClient(logger, null);	
		ArrayList<DetailsTuple> detailsToLog = null;	
		
		RecordProcessResults retval = new RecordProcessResults();
		
		try {
			if (trackFlag != null && trackFlag) {
				// validate tracker before starting anything
				this.validateTracker();
			}
			if(logger != null){ 
				detailsToLog = LoggerRestClient.addDetails("Ingestion Type", "From CSV", null);
			}
			
			// get SparqlGraphJson from template
			String templateContent = fromFiles ? new String(((MultipartFile)templateFile).getBytes()) : (String)templateFile;
			if(templateContent != null){
				LocalLogger.logToStdErr("template size: "  + templateContent.length());
			}else{
				LocalLogger.logToStdErr("template content was null");
			}	
			
			SparqlGraphJson sgJson = new SparqlGraphJson(Utility.getJsonObjectFromString(templateContent));			
			
			// get data file content
			String dataFileContent = fromFiles ? new String(((MultipartFile)dataFile).getBytes()) : (String)dataFile; 
			String dataFileName = fromFiles ? ((MultipartFile)dataFile).getName() : null;
			if(dataFileContent != null){
				LocalLogger.logToStdErr("data size: "  + dataFileContent.length());
			}else{
				LocalLogger.logToStdErr("data content was null");
			}
					
			// override the connection, if needed
			if(sparqlConnectionOverride != null){
				String sparqlConnectionString = fromFiles ? new String(((MultipartFile)sparqlConnectionOverride).getBytes()) : (String)sparqlConnectionOverride;				
				sgJson.setSparqlConn( new SparqlConnection(sparqlConnectionString));   				
			}
						
			if(logger != null){  
				detailsToLog = LoggerRestClient.addDetails("template", templateContent, detailsToLog);
			}
					
			// get a CSV data set to use in the load. 
			Dataset ds = new CSVDataset(dataFileContent, true);

			// how about some more logging
			String startTime = dateFormat.format(Calendar.getInstance().getTime());
			if(logger != null) { 
				detailsToLog = LoggerRestClient.addDetails("Start Time", startTime, detailsToLog); 				
			}
			
			String trackKey = UUID.randomUUID().toString();
			this.overrideBaseURI(sgJson, trackFlag, overrideBaseURI, trackKey);

			// clear caches
			uncache(sgJson.getSparqlConn().getInsertInterface());
			
			// load
			DataLoader dl = new DataLoader(sgJson, ds, prop.getSparqlUserName(), prop.getSparqlPassword());
			dl.overrideMaxThreads(prop.getMaxThreads());

			recordsProcessed = dl.importData(precheck, skipIngest);
	
			// yet some more logging
			String endTime = dateFormat.format(Calendar.getInstance().getTime());
			if(logger != null) { 
				detailsToLog = LoggerRestClient.addDetails("End Time", endTime, detailsToLog); 
				detailsToLog = LoggerRestClient.addDetails("input record size", recordsProcessed + "", detailsToLog);	
			}
			
			// set success values
			if(precheck && dl.getLoadingErrorReport().getRows().size() == 0){
				if (trackFlag != null && trackFlag) {
					this.trackLoad(trackKey, dataFileName, dataFileContent, sgJson.getSparqlConn().getInsertInterface());
				}
				retval.setSuccess(true);
			} else if(precheck && dl.getLoadingErrorReport().getRows().size() != 0){
				retval.setSuccess(false);
			} else if(!precheck && recordsProcessed > 0){
				if (trackFlag != null && trackFlag) {
					this.trackLoad(trackKey, dataFileName, dataFileContent, sgJson.getSparqlConn().getInsertInterface());
				}
				retval.setSuccess(true);
			} else {
				retval.setSuccess(false);
			}			
			
			retval.setRecordsProcessed(recordsProcessed);
			retval.setFailuresEncountered(dl.getLoadingErrorReport().getRows().size());
			retval.addResults(dl.getLoadingErrorReport());
		} catch (Exception e) {
			// TODO write failure JSONObject to return and return it.
			LocalLogger.printStackTrace(e);			
			retval.setSuccess(false);
			retval.addRationaleMessage("ingestion", "fromCsv*", e);
		}  
		
		if(logger != null){  
			// what are we returning
			detailsToLog = LoggerRestClient.addDetails("error code", retval.getResultCodeString(), detailsToLog);
			detailsToLog = LoggerRestClient.addDetails("results", retval.toJson().toJSONString(), detailsToLog);
			detailsToLog = LoggerRestClient.addDetails("records Processed", recordsProcessed + "", detailsToLog);
		}
		
		if(logger != null){ 
			logger.logEvent("Data ingestion", detailsToLog, "Add Instance Data To Triple Store");
		}
		return retval.toJson();
	}
/**
	 * Load data from csv.
	 * @param templateFile the json template (File if fromFiles=true, else String)
	 * @param dataFile the data file (File if fromFiles=true, else String)
	 * @param sparqlConnectionOverride SPARQL connection json (File if fromFiles=true, else String)  If non-null, will override the connection in the template.
	 * @param fromFiles true to indicate that the 3 above parameters are Files, else Strings
	 * @param precheck check that the ingest will succeed before starting it
	 * @param skipIngest skip the actual ingest (e.g. for precheck only)
	 * @param async perform async
	 */
Load data from csv.
@param templateFile the json template (File if fromFiles=true, else String)
@param dataFile the data file (File if fromFiles=true, else String)
@param sparqlConnectionOverride SPARQL connection json (File if fromFiles=true, else String)  If non-null, will override the connection in the template.
@param fromFiles true to indicate that the 3 above parameters are Files, else Strings
@param precheck check that the ingest will succeed before starting it
@param skipIngest skip the actual ingest
@param async perform async
Load data from csv . @param templateFile the json template ( File if fromFiles = true else String ) @param dataFile the data file ( File if fromFiles = true else String ) @param sparqlConnectionOverride SPARQL connection json ( File if fromFiles = true else String ) If non - null will override the connection in the template . @param fromFiles true to indicate that the 3 above parameters are Files else Strings @param precheck check that the ingest will succeed before starting it @param skipIngest skip the actual ingest @param async perform async
private JSONObject fromAnyCsv(Object templateFile, Object dataFile, Object sparqlConnectionOverride, Boolean fromFiles, Boolean precheck, Boolean skipIngest, Boolean trackFlag, String overrideBaseURI){
		DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
		int recordsProcessed = 0;
		LoggerRestClient logger = null;
		logger = getLoggerRestClient(logger, null);	
		ArrayList<DetailsTuple> detailsToLog = null;	
		RecordProcessResults retval = new RecordProcessResults();
		try {
			if (trackFlag != null && trackFlag) {
				this.validateTracker();
			}
			if(logger != null){ 
				detailsToLog = LoggerRestClient.addDetails("Ingestion Type", "From CSV", null);
			}
			String templateContent = fromFiles ? new String(((MultipartFile)templateFile).getBytes()) : (String)templateFile;
			if(templateContent != null){
				LocalLogger.logToStdErr("template size: "  + templateContent.length());
			}else{
				LocalLogger.logToStdErr("template content was null");
			}	
			SparqlGraphJson sgJson = new SparqlGraphJson(Utility.getJsonObjectFromString(templateContent));			
			String dataFileContent = fromFiles ? new String(((MultipartFile)dataFile).getBytes()) : (String)dataFile; 
			String dataFileName = fromFiles ? ((MultipartFile)dataFile).getName() : null;
			if(dataFileContent != null){
				LocalLogger.logToStdErr("data size: "  + dataFileContent.length());
			}else{
				LocalLogger.logToStdErr("data content was null");
			}
			if(sparqlConnectionOverride != null){
				String sparqlConnectionString = fromFiles ? new String(((MultipartFile)sparqlConnectionOverride).getBytes()) : (String)sparqlConnectionOverride;				
				sgJson.setSparqlConn( new SparqlConnection(sparqlConnectionString));   				
			}
			if(logger != null){  
				detailsToLog = LoggerRestClient.addDetails("template", templateContent, detailsToLog);
			}
			Dataset ds = new CSVDataset(dataFileContent, true);
			String startTime = dateFormat.format(Calendar.getInstance().getTime());
			if(logger != null) { 
				detailsToLog = LoggerRestClient.addDetails("Start Time", startTime, detailsToLog); 				
			}
			String trackKey = UUID.randomUUID().toString();
			this.overrideBaseURI(sgJson, trackFlag, overrideBaseURI, trackKey);
			uncache(sgJson.getSparqlConn().getInsertInterface());
			DataLoader dl = new DataLoader(sgJson, ds, prop.getSparqlUserName(), prop.getSparqlPassword());
			dl.overrideMaxThreads(prop.getMaxThreads());
			recordsProcessed = dl.importData(precheck, skipIngest);
			String endTime = dateFormat.format(Calendar.getInstance().getTime());
			if(logger != null) { 
				detailsToLog = LoggerRestClient.addDetails("End Time", endTime, detailsToLog); 
				detailsToLog = LoggerRestClient.addDetails("input record size", recordsProcessed + "", detailsToLog);	
			}
			if(precheck && dl.getLoadingErrorReport().getRows().size() == 0){
				if (trackFlag != null && trackFlag) {
					this.trackLoad(trackKey, dataFileName, dataFileContent, sgJson.getSparqlConn().getInsertInterface());
				}
				retval.setSuccess(true);
			} else if(precheck && dl.getLoadingErrorReport().getRows().size() != 0){
				retval.setSuccess(false);
			} else if(!precheck && recordsProcessed > 0){
				if (trackFlag != null && trackFlag) {
					this.trackLoad(trackKey, dataFileName, dataFileContent, sgJson.getSparqlConn().getInsertInterface());
				}
				retval.setSuccess(true);
			} else {
				retval.setSuccess(false);
			}			
			retval.setRecordsProcessed(recordsProcessed);
			retval.setFailuresEncountered(dl.getLoadingErrorReport().getRows().size());
			retval.addResults(dl.getLoadingErrorReport());
		} catch (Exception e) {
			LocalLogger.printStackTrace(e);			
			retval.setSuccess(false);
			retval.addRationaleMessage("ingestion", "fromCsv*", e);
		}  
		if(logger != null){  
			detailsToLog = LoggerRestClient.addDetails("error code", retval.getResultCodeString(), detailsToLog);
			detailsToLog = LoggerRestClient.addDetails("results", retval.toJson().toJSONString(), detailsToLog);
			detailsToLog = LoggerRestClient.addDetails("records Processed", recordsProcessed + "", detailsToLog);
		}
		if(logger != null){ 
			logger.logEvent("Data ingestion", detailsToLog, "Add Instance Data To Triple Store");
		}
		return retval.toJson();
	}
private JSONObject fromAnyCsv ( Object templateFile , Object dataFile , Object sparqlConnectionOverride , Boolean fromFiles , Boolean precheck , Boolean skipIngest , Boolean trackFlag , String overrideBaseURI ) { DateFormat dateFormat = new SimpleDateFormat ( "yyyy/MM/dd HH:mm:ss" ) ; int recordsProcessed = 0 ; LoggerRestClient logger = null ; logger = getLoggerRestClient ( logger , null ) ; ArrayList < DetailsTuple > detailsToLog = null ; RecordProcessResults retval = new RecordProcessResults ( ) ; try { if ( trackFlag != null && trackFlag ) { this . validateTracker ( ) ; } if ( logger != null ) { detailsToLog = LoggerRestClient . addDetails ( "Ingestion Type" , "From CSV" , null ) ; } String templateContent = fromFiles ? new String ( ( ( MultipartFile ) templateFile ) . getBytes ( ) ) : ( String ) templateFile ; if ( templateContent != null ) { LocalLogger . logToStdErr ( "template size: " + templateContent . length ( ) ) ; } else { LocalLogger . logToStdErr ( "template content was null" ) ; } SparqlGraphJson sgJson = new SparqlGraphJson ( Utility . getJsonObjectFromString ( templateContent ) ) ; String dataFileContent = fromFiles ? new String ( ( ( MultipartFile ) dataFile ) . getBytes ( ) ) : ( String ) dataFile ; String dataFileName = fromFiles ? ( ( MultipartFile ) dataFile ) . getName ( ) : null ; if ( dataFileContent != null ) { LocalLogger . logToStdErr ( "data size: " + dataFileContent . length ( ) ) ; } else { LocalLogger . logToStdErr ( "data content was null" ) ; } if ( sparqlConnectionOverride != null ) { String sparqlConnectionString = fromFiles ? new String ( ( ( MultipartFile ) sparqlConnectionOverride ) . getBytes ( ) ) : ( String ) sparqlConnectionOverride ; sgJson . setSparqlConn ( new SparqlConnection ( sparqlConnectionString ) ) ; } if ( logger != null ) { detailsToLog = LoggerRestClient . addDetails ( "template" , templateContent , detailsToLog ) ; } Dataset ds = new CSVDataset ( dataFileContent , true ) ; String startTime = dateFormat . format ( Calendar . getInstance ( ) . getTime ( ) ) ; if ( logger != null ) { detailsToLog = LoggerRestClient . addDetails ( "Start Time" , startTime , detailsToLog ) ; } String trackKey = UUID . randomUUID ( ) . toString ( ) ; this . overrideBaseURI ( sgJson , trackFlag , overrideBaseURI , trackKey ) ; uncache ( sgJson . getSparqlConn ( ) . getInsertInterface ( ) ) ; DataLoader dl = new DataLoader ( sgJson , ds , prop . getSparqlUserName ( ) , prop . getSparqlPassword ( ) ) ; dl . overrideMaxThreads ( prop . getMaxThreads ( ) ) ; recordsProcessed = dl . importData ( precheck , skipIngest ) ; String endTime = dateFormat . format ( Calendar . getInstance ( ) . getTime ( ) ) ; if ( logger != null ) { detailsToLog = LoggerRestClient . addDetails ( "End Time" , endTime , detailsToLog ) ; detailsToLog = LoggerRestClient . addDetails ( "input record size" , recordsProcessed + "" , detailsToLog ) ; } if ( precheck && dl . getLoadingErrorReport ( ) . getRows ( ) . size ( ) == 0 ) { if ( trackFlag != null && trackFlag ) { this . trackLoad ( trackKey , dataFileName , dataFileContent , sgJson . getSparqlConn ( ) . getInsertInterface ( ) ) ; } retval . setSuccess ( true ) ; } else if ( precheck && dl . getLoadingErrorReport ( ) . getRows ( ) . size ( ) != 0 ) { retval . setSuccess ( false ) ; } else if ( ! precheck && recordsProcessed > 0 ) { if ( trackFlag != null && trackFlag ) { this . trackLoad ( trackKey , dataFileName , dataFileContent , sgJson . getSparqlConn ( ) . getInsertInterface ( ) ) ; } retval . setSuccess ( true ) ; } else { retval . setSuccess ( false ) ; } retval . setRecordsProcessed ( recordsProcessed ) ; retval . setFailuresEncountered ( dl . getLoadingErrorReport ( ) . getRows ( ) . size ( ) ) ; retval . addResults ( dl . getLoadingErrorReport ( ) ) ; } catch ( Exception e ) { LocalLogger . printStackTrace ( e ) ; retval . setSuccess ( false ) ; retval . addRationaleMessage ( "ingestion" , "fromCsv*" , e ) ; } if ( logger != null ) { detailsToLog = LoggerRestClient . addDetails ( "error code" , retval . getResultCodeString ( ) , detailsToLog ) ; detailsToLog = LoggerRestClient . addDetails ( "results" , retval . toJson ( ) . toJSONString ( ) , detailsToLog ) ; detailsToLog = LoggerRestClient . addDetails ( "records Processed" , recordsProcessed + "" , detailsToLog ) ; } if ( logger != null ) { logger . logEvent ( "Data ingestion" , detailsToLog , "Add Instance Data To Triple Store" ) ; } return retval . toJson ( ) ; }
Load data from csv.
Load data from csv .
// set up the logger // validate tracker before starting anything // get SparqlGraphJson from template // get data file content // override the connection, if needed // get a CSV data set to use in the load.  // how about some more logging // clear caches // load // yet some more logging // set success values // TODO write failure JSONObject to return and return it. // what are we returning