sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/utility/ImportSpecHandler.java
public NodeGroup buildImportNodegroup(ArrayList<String> record, boolean skipValidation) throws Exception{

		// create a new nodegroup copy. 
		NodeGroup retNodegroup = NodeGroup.getInstanceFromJson(this.nodegroupJson);
		retNodegroup.clearOrderBy();
		
		if(record  == null){ throw new Exception("incoming record cannot be null for ImportSpecHandler.getValues"); }
		if(this.colNameToIndexHash.isEmpty()){ throw new Exception("the header positions were never set for the importspechandler"); }
		
		// fill in all URI's, possibly with NOT_FOUND if that's legal
		try {
			this.lookupAllUris(retNodegroup, record);
		} catch (Exception e) {
			// swallow URI lookup exception if we're skipping validation
			if (!skipValidation) {
				throw e;
			}
		}
		
		// do mappings
		for (int i=0; i < this.importMappings.length; i++) {
			this.addMappingToNodegroup(retNodegroup, this.importMappings[i], record, skipValidation);
		}
		
		// do lookupMappings for any URI that was just generated - creating so fill it in
		// also fill in EMPTY-LOOKUPs so they don't get pruned.  This will throw an error later: empty lookup column
		for (int i=0; i < retNodegroup.getNodeCount(); i++) {
			Node n = retNodegroup.getNode(i);
			
			// Fixed a bug here by adding this.uriCache.isEmptyLookup() condition.
			// EMPTY-LOOKUPs should have any other mappings added so that error handling works later
			// If all mappings are empty then the empty node might be prunable and the ingestion succeeds without it.
			// See git commit 9/24/2021
			// See DataLoaderTest_IT.testMissingURILookups()
			if (this.uriCache.isGenerated(n.getInstanceValue()) || this.uriCache.isEmptyLookup(n.getInstanceValue())) {   
				for (ImportMapping lookupMapping : this.lookupMappings.get(n.getSparqlID())) {
					this.addMappingToNodegroup(retNodegroup,  lookupMapping, record, skipValidation);
				}
			}
		}
		
		// prune nodes that no longer belong (no uri and no properties)
		this.pruneImportNodegroup(retNodegroup);
		
		// set URI for nulls
		retNodegroup = this.setURIsForNullNodes(retNodegroup);
		return retNodegroup;
	}
/**
	 * Create a nodegroup to ingest a single record
	 * Always performs URILookup (checking UriCache first), possibly setting to NOT_FOUND
	 * 
	 * In order to accommodate threading and generation of unknown Uris,
	 * this function must always be called twice:
	 * Option 1:  "pre-check"
	 * 		buildImportNodegroup(record, false)  // do validation and URI lookup.  Threaded if desired.
	 *                                           // Caller join threads and make sure all validations passed, or quit.
	 *      generateNotFoundURIs()               // generate any legally not found URI's after validation threads are joined
	 *      buildImportNodegroup(record, true)   // quicker generation using cached URI's and no validation
	 *                                           // This can be threaded
     *                                           // caller should do the insert on the nodegroup
	 *                                           
	 * Option 2: "single pass"
	 * 		buildImportNodegroup(record, true)   // No validation of types.  URI lookup with silent failure. Threaded if desired.    
	 *                                           // Even though not validating, this pass is needed to discover NOT_FOUND Uri's                
	 *      generateNotFoundURIs()               // generate any legally not found URI's after threads are joined
	 *      buildImportNodegroup(record, false)  // validation and nodegroup generation using cached URI's
	 *                                           // This can be threaded
	 *                                           // caller should insert nodegroup.  Incomplete load possible.
	 *                                           // caller report any incomplete loads                                        
	 * @param record
	 * @param skipValidation
	 * @return
	 * @throws Exception
	 */
Create a nodegroup to ingest a single record
Always performs URILookup (checking UriCache first), possibly setting to NOT_FOUND

In order to accommodate threading and generation of unknown Uris,
this function must always be called twice:
Option 1:  "pre-check"
buildImportNodegroup(record, false)  // do validation and URI lookup.  Threaded if desired.
Caller join threads and make sure all validations passed, or quit.
generateNotFoundURIs()               // generate any legally not found URI's after validation threads are joined
buildImportNodegroup(record, true)   // quicker generation using cached URI's and no validation
This can be threaded
caller should do the insert on the nodegroup

Option 2: "single pass"
buildImportNodegroup(record, true)   // No validation of types.  URI lookup with silent failure. Threaded if desired.
Even though not validating, this pass is needed to discover NOT_FOUND Uri's
generateNotFoundURIs()               // generate any legally not found URI's after threads are joined
buildImportNodegroup(record, false)  // validation and nodegroup generation using cached URI's
This can be threaded
caller should insert nodegroup.  Incomplete load possible.
caller report any incomplete loads
@param record
@param skipValidation
@return
@throws Exception
Create a nodegroup to ingest a single record Always performs URILookup ( checking UriCache first ) possibly setting to NOT_FOUND In order to accommodate threading and generation of unknown Uris this function must always be called twice : Option 1 : " pre - check " buildImportNodegroup ( record false ) // do validation and URI lookup . Threaded if desired . Caller join threads and make sure all validations passed or quit . generateNotFoundURIs () // generate any legally not found URI ' s after validation threads are joined buildImportNodegroup ( record true ) // quicker generation using cached URI ' s and no validation This can be threaded caller should do the insert on the nodegroup Option 2 : " single pass " buildImportNodegroup ( record true ) // No validation of types . URI lookup with silent failure . Threaded if desired . Even though not validating this pass is needed to discover NOT_FOUND Uri ' s generateNotFoundURIs () // generate any legally not found URI ' s after threads are joined buildImportNodegroup ( record false ) // validation and nodegroup generation using cached URI ' s This can be threaded caller should insert nodegroup . Incomplete load possible . caller report any incomplete loads @param record @param skipValidation @return @throws Exception
public NodeGroup buildImportNodegroup(ArrayList<String> record, boolean skipValidation) throws Exception{
		NodeGroup retNodegroup = NodeGroup.getInstanceFromJson(this.nodegroupJson);
		retNodegroup.clearOrderBy();
		if(record  == null){ throw new Exception("incoming record cannot be null for ImportSpecHandler.getValues"); }
		if(this.colNameToIndexHash.isEmpty()){ throw new Exception("the header positions were never set for the importspechandler"); }
		try {
			this.lookupAllUris(retNodegroup, record);
		} catch (Exception e) {
			if (!skipValidation) {
				throw e;
			}
		}
		for (int i=0; i < this.importMappings.length; i++) {
			this.addMappingToNodegroup(retNodegroup, this.importMappings[i], record, skipValidation);
		}
		for (int i=0; i < retNodegroup.getNodeCount(); i++) {
			Node n = retNodegroup.getNode(i);
			if (this.uriCache.isGenerated(n.getInstanceValue()) || this.uriCache.isEmptyLookup(n.getInstanceValue())) {   
				for (ImportMapping lookupMapping : this.lookupMappings.get(n.getSparqlID())) {
					this.addMappingToNodegroup(retNodegroup,  lookupMapping, record, skipValidation);
				}
			}
		}
		this.pruneImportNodegroup(retNodegroup);
		retNodegroup = this.setURIsForNullNodes(retNodegroup);
		return retNodegroup;
	}
public NodeGroup buildImportNodegroup ( ArrayList < String > record , boolean skipValidation ) throws Exception { NodeGroup retNodegroup = NodeGroup . getInstanceFromJson ( this . nodegroupJson ) ; retNodegroup . clearOrderBy ( ) ; if ( record == null ) { throw new Exception ( "incoming record cannot be null for ImportSpecHandler.getValues" ) ; } if ( this . colNameToIndexHash . isEmpty ( ) ) { throw new Exception ( "the header positions were never set for the importspechandler" ) ; } try { this . lookupAllUris ( retNodegroup , record ) ; } catch ( Exception e ) { if ( ! skipValidation ) { throw e ; } } for ( int i = 0 ; i < this . importMappings . length ; i ++ ) { this . addMappingToNodegroup ( retNodegroup , this . importMappings [ i ] , record , skipValidation ) ; } for ( int i = 0 ; i < retNodegroup . getNodeCount ( ) ; i ++ ) { Node n = retNodegroup . getNode ( i ) ; if ( this . uriCache . isGenerated ( n . getInstanceValue ( ) ) || this . uriCache . isEmptyLookup ( n . getInstanceValue ( ) ) ) { for ( ImportMapping lookupMapping : this . lookupMappings . get ( n . getSparqlID ( ) ) ) { this . addMappingToNodegroup ( retNodegroup , lookupMapping , record , skipValidation ) ; } } } this . pruneImportNodegroup ( retNodegroup ) ; retNodegroup = this . setURIsForNullNodes ( retNodegroup ) ; return retNodegroup ; }
Create a nodegroup to ingest a single record
 Always performs URILookup (checking UriCache first), possibly setting to NOT_FOUND
Create a nodegroup to ingest a single record Always performs URILookup ( checking UriCache first ) possibly setting to NOT_FOUND
// create a new nodegroup copy.  // fill in all URI's, possibly with NOT_FOUND if that's legal // swallow URI lookup exception if we're skipping validation // do mappings // do lookupMappings for any URI that was just generated - creating so fill it in // also fill in EMPTY-LOOKUPs so they don't get pruned.  This will throw an error later: empty lookup column // Fixed a bug here by adding this.uriCache.isEmptyLookup() condition. // EMPTY-LOOKUPs should have any other mappings added so that error handling works later // If all mappings are empty then the empty node might be prunable and the ingestion succeeds without it. // See git commit 9/24/2021 // See DataLoaderTest_IT.testMissingURILookups() // prune nodes that no longer belong (no uri and no properties) // set URI for nulls