MRFramework/src/main/java/com/mr/util/AWSUtility.java
/**
	 * Reads the input folder in S3 and creates file names list for each node
	 * 
	 * @param bucket : S3 bucket name 
	 * @param folder : folder from where files are to be read
	 * @returns List of file names for each node
	 *  
	 * @throws IOException
	 */
Reads the input folder in S3 and creates file names list for each node

@param bucket : S3 bucket name
@param folder : folder from where files are to be read
@returns List of file names for each node


Reads the input folder in S3 and creates file names list for each node @param bucket : S3 bucket name @param folder : folder from where files are to be read @returns List of file names for each node
@SuppressWarnings("unchecked")
	public static List<String>[] s3ReadBuckets(String bucket, String folder) throws IOException {
		ObjectListing objs = App.s3.listObjects(new ListObjectsRequest().withBucketName(bucket).withPrefix(folder));
		Integer fileCount = 0;
		int nodes = App.ipmap.size();
		System.out.println("nodes = "+ nodes);
		List<String>[] filesForNodes = new ArrayList[nodes]; 
		for (S3ObjectSummary obj : objs.getObjectSummaries()) {
			List<String> files = filesForNodes[fileCount%nodes];
			if(files == null) files = new ArrayList<String>();
			files.add(obj.getKey());
			filesForNodes[fileCount%nodes] = files;
			fileCount++;
		}
		return filesForNodes;
	}
@ SuppressWarnings ( "unchecked" ) public static List < String > [ ] s3ReadBuckets ( String bucket , String folder ) throws IOException { ObjectListing objs = App . s3 . listObjects ( new ListObjectsRequest ( ) . withBucketName ( bucket ) . withPrefix ( folder ) ) ; Integer fileCount = 0 ; int nodes = App . ipmap . size ( ) ; System . out . println ( "nodes = " + nodes ) ; List < String > [ ] filesForNodes = new ArrayList [ nodes ] ; for ( S3ObjectSummary obj : objs . getObjectSummaries ( ) ) { List < String > files = filesForNodes [ fileCount % nodes ] ; if ( files == null ) files = new ArrayList < String > ( ) ; files . add ( obj . getKey ( ) ) ; filesForNodes [ fileCount % nodes ] = files ; fileCount ++ ; } return filesForNodes ; }
Reads the input folder in S3 and creates file names list for each node
Reads the input folder in S3 and creates file names list for each node