sparqlGraphIngestionService/src/main/java/com/ge/research/semtk/services/ingestion/IngestionRestController.java
private SimpleResultSet fromAnyCsvAsync(Object templateFile, Object dataFile, Object sparqlConnectionOverride, Boolean fromFiles, Boolean precheck, Boolean skipIngest, Boolean trackFlag, String overrideBaseURI){

		// TODO, difficult to loadTrack because we don't know success
		int recordsProcessed = 0;
		SimpleResultSet simpleResult = new SimpleResultSet();
		
		// set up the logger
		LoggerRestClient logger = LoggerRestClient.getInstance(prop, ThreadAuthenticator.getThreadUserName());
				
		try {
			
			if (trackFlag != null && trackFlag) {
				// validate tracker before starting anything
				this.validateTracker();
			}
			
			// get SparqlGraphJson from template
			String templateContent = fromFiles ? new String(((MultipartFile)templateFile).getBytes()) : (String)templateFile;	
			SparqlGraphJson sgJson = new SparqlGraphJson(Utility.getJsonObjectFromString(templateContent));			
			
			// get data file content
			String dataFileContent = fromFiles ? new String(((MultipartFile)dataFile).getBytes()) : (String)dataFile;
			LoggerRestClient.easyLog(logger, SERVICE_NAME, "fromAnyCsvAsync", "chars", String.valueOf(dataFileContent != null ? dataFileContent.length() : 0));
		
			// override the connection, if needed
			if(sparqlConnectionOverride != null){
				String sparqlConnectionString = fromFiles ? new String(((MultipartFile)sparqlConnectionOverride).getBytes()) : (String)sparqlConnectionOverride;				
				sgJson.setSparqlConn( new SparqlConnection(sparqlConnectionString));   				
			}
					
			String trackKey = UUID.randomUUID().toString();
			String jobId = "job-" + UUID.randomUUID().toString();
			this.overrideBaseURI(sgJson, trackFlag, overrideBaseURI, trackKey);
			// get a CSV data set to use in the load. 
			Dataset ds = new CSVDataset(dataFileContent, true);
			DataLoader dl = new DataLoader(sgJson, ds, prop.getSparqlUserName(), prop.getSparqlPassword());
			dl.overrideMaxThreads(prop.getMaxThreads());
			
			
			dl.runAsync(precheck, skipIngest, 
					new StatusClient(new StatusClientConfig(status_prop.getProtocol(), status_prop.getServer(), status_prop.getPort(), jobId)), 
					new ResultsClient(new ResultsClientConfig(results_prop.getProtocol(), results_prop.getServer(), results_prop.getPort()))
					);
			
			if (trackFlag != null && trackFlag) {
				AsyncLoadTrackThread thread = new AsyncLoadTrackThread(
						new JobTracker(servicesgraph_prop.buildSei()),
						jobId, tracker, trackBucket, trackKey, 
						((MultipartFile)dataFile).getName(), 
						sgJson.getSparqlConn().getInsertInterface(), 
						dataFileContent.getBytes());
				thread.start();
			}
			
			simpleResult.setSuccess(true);
			simpleResult.addResult(SimpleResultSet.JOB_ID_RESULT_KEY, jobId);
					
			
		} catch (Exception e) {
			// TODO write failure JSONObject to return and return it.
			LoggerRestClient.easyLog(logger, SERVICE_NAME, "fromAnyCsvAsync exception", "message", e.toString());
			LocalLogger.printStackTrace(e);			
			simpleResult.setSuccess(false);
			simpleResult.addRationaleMessage("ingestion", "fromCsv*", e);
		}  
		
		return simpleResult;
	}
/**
	 * Load data from csv.
	 * Note this is a newer copy of fromAsynCsv with Async added
	 *     - more "modern" logging
	 * @param templateFile the json template (File if fromFiles=true, else String)
	 * @param dataFile the data file (File if fromFiles=true, else String)
	 * @param sparqlConnectionOverride SPARQL connection json (File if fromFiles=true, else String)  If non-null, will override the connection in the template.
	 * @param fromFiles true to indicate that the 3 above parameters are Files, else Strings
	 * @param precheck check that the ingest will succeed before starting it
	 * @param skipIngest skip the actual ingest (e.g. for precheck only)
	 */
Load data from csv.
Note this is a newer copy of fromAsynCsv with Async added
- more "modern" logging
@param templateFile the json template (File if fromFiles=true, else String)
@param dataFile the data file (File if fromFiles=true, else String)
@param sparqlConnectionOverride SPARQL connection json (File if fromFiles=true, else String)  If non-null, will override the connection in the template.
@param fromFiles true to indicate that the 3 above parameters are Files, else Strings
@param precheck check that the ingest will succeed before starting it
@param skipIngest skip the actual ingest
Load data from csv . Note this is a newer copy of fromAsynCsv with Async added - more " modern " logging @param templateFile the json template ( File if fromFiles = true else String ) @param dataFile the data file ( File if fromFiles = true else String ) @param sparqlConnectionOverride SPARQL connection json ( File if fromFiles = true else String ) If non - null will override the connection in the template . @param fromFiles true to indicate that the 3 above parameters are Files else Strings @param precheck check that the ingest will succeed before starting it @param skipIngest skip the actual ingest
private SimpleResultSet fromAnyCsvAsync(Object templateFile, Object dataFile, Object sparqlConnectionOverride, Boolean fromFiles, Boolean precheck, Boolean skipIngest, Boolean trackFlag, String overrideBaseURI){
		int recordsProcessed = 0;
		SimpleResultSet simpleResult = new SimpleResultSet();
		LoggerRestClient logger = LoggerRestClient.getInstance(prop, ThreadAuthenticator.getThreadUserName());
		try {
			if (trackFlag != null && trackFlag) {
				this.validateTracker();
			}
			String templateContent = fromFiles ? new String(((MultipartFile)templateFile).getBytes()) : (String)templateFile;	
			SparqlGraphJson sgJson = new SparqlGraphJson(Utility.getJsonObjectFromString(templateContent));			
			String dataFileContent = fromFiles ? new String(((MultipartFile)dataFile).getBytes()) : (String)dataFile;
			LoggerRestClient.easyLog(logger, SERVICE_NAME, "fromAnyCsvAsync", "chars", String.valueOf(dataFileContent != null ? dataFileContent.length() : 0));
			if(sparqlConnectionOverride != null){
				String sparqlConnectionString = fromFiles ? new String(((MultipartFile)sparqlConnectionOverride).getBytes()) : (String)sparqlConnectionOverride;				
				sgJson.setSparqlConn( new SparqlConnection(sparqlConnectionString));   				
			}
			String trackKey = UUID.randomUUID().toString();
			String jobId = "job-" + UUID.randomUUID().toString();
			this.overrideBaseURI(sgJson, trackFlag, overrideBaseURI, trackKey);
			Dataset ds = new CSVDataset(dataFileContent, true);
			DataLoader dl = new DataLoader(sgJson, ds, prop.getSparqlUserName(), prop.getSparqlPassword());
			dl.overrideMaxThreads(prop.getMaxThreads());
			dl.runAsync(precheck, skipIngest, 
					new StatusClient(new StatusClientConfig(status_prop.getProtocol(), status_prop.getServer(), status_prop.getPort(), jobId)), 
					new ResultsClient(new ResultsClientConfig(results_prop.getProtocol(), results_prop.getServer(), results_prop.getPort()))
					);
			if (trackFlag != null && trackFlag) {
				AsyncLoadTrackThread thread = new AsyncLoadTrackThread(
						new JobTracker(servicesgraph_prop.buildSei()),
						jobId, tracker, trackBucket, trackKey, 
						((MultipartFile)dataFile).getName(), 
						sgJson.getSparqlConn().getInsertInterface(), 
						dataFileContent.getBytes());
				thread.start();
			}
			simpleResult.setSuccess(true);
			simpleResult.addResult(SimpleResultSet.JOB_ID_RESULT_KEY, jobId);
		} catch (Exception e) {
			LoggerRestClient.easyLog(logger, SERVICE_NAME, "fromAnyCsvAsync exception", "message", e.toString());
			LocalLogger.printStackTrace(e);			
			simpleResult.setSuccess(false);
			simpleResult.addRationaleMessage("ingestion", "fromCsv*", e);
		}  
		return simpleResult;
	}
private SimpleResultSet fromAnyCsvAsync ( Object templateFile , Object dataFile , Object sparqlConnectionOverride , Boolean fromFiles , Boolean precheck , Boolean skipIngest , Boolean trackFlag , String overrideBaseURI ) { int recordsProcessed = 0 ; SimpleResultSet simpleResult = new SimpleResultSet ( ) ; LoggerRestClient logger = LoggerRestClient . getInstance ( prop , ThreadAuthenticator . getThreadUserName ( ) ) ; try { if ( trackFlag != null && trackFlag ) { this . validateTracker ( ) ; } String templateContent = fromFiles ? new String ( ( ( MultipartFile ) templateFile ) . getBytes ( ) ) : ( String ) templateFile ; SparqlGraphJson sgJson = new SparqlGraphJson ( Utility . getJsonObjectFromString ( templateContent ) ) ; String dataFileContent = fromFiles ? new String ( ( ( MultipartFile ) dataFile ) . getBytes ( ) ) : ( String ) dataFile ; LoggerRestClient . easyLog ( logger , SERVICE_NAME , "fromAnyCsvAsync" , "chars" , String . valueOf ( dataFileContent != null ? dataFileContent . length ( ) : 0 ) ) ; if ( sparqlConnectionOverride != null ) { String sparqlConnectionString = fromFiles ? new String ( ( ( MultipartFile ) sparqlConnectionOverride ) . getBytes ( ) ) : ( String ) sparqlConnectionOverride ; sgJson . setSparqlConn ( new SparqlConnection ( sparqlConnectionString ) ) ; } String trackKey = UUID . randomUUID ( ) . toString ( ) ; String jobId = "job-" + UUID . randomUUID ( ) . toString ( ) ; this . overrideBaseURI ( sgJson , trackFlag , overrideBaseURI , trackKey ) ; Dataset ds = new CSVDataset ( dataFileContent , true ) ; DataLoader dl = new DataLoader ( sgJson , ds , prop . getSparqlUserName ( ) , prop . getSparqlPassword ( ) ) ; dl . overrideMaxThreads ( prop . getMaxThreads ( ) ) ; dl . runAsync ( precheck , skipIngest , new StatusClient ( new StatusClientConfig ( status_prop . getProtocol ( ) , status_prop . getServer ( ) , status_prop . getPort ( ) , jobId ) ) , new ResultsClient ( new ResultsClientConfig ( results_prop . getProtocol ( ) , results_prop . getServer ( ) , results_prop . getPort ( ) ) ) ) ; if ( trackFlag != null && trackFlag ) { AsyncLoadTrackThread thread = new AsyncLoadTrackThread ( new JobTracker ( servicesgraph_prop . buildSei ( ) ) , jobId , tracker , trackBucket , trackKey , ( ( MultipartFile ) dataFile ) . getName ( ) , sgJson . getSparqlConn ( ) . getInsertInterface ( ) , dataFileContent . getBytes ( ) ) ; thread . start ( ) ; } simpleResult . setSuccess ( true ) ; simpleResult . addResult ( SimpleResultSet . JOB_ID_RESULT_KEY , jobId ) ; } catch ( Exception e ) { LoggerRestClient . easyLog ( logger , SERVICE_NAME , "fromAnyCsvAsync exception" , "message" , e . toString ( ) ) ; LocalLogger . printStackTrace ( e ) ; simpleResult . setSuccess ( false ) ; simpleResult . addRationaleMessage ( "ingestion" , "fromCsv*" , e ) ; } return simpleResult ; }
Load data from csv.
Load data from csv .
// TODO, difficult to loadTrack because we don't know success // set up the logger // validate tracker before starting anything // get SparqlGraphJson from template // get data file content // override the connection, if needed // get a CSV data set to use in the load.  // TODO write failure JSONObject to return and return it.