src/core/preprocess/ReadTextFile.java
public static HashMap<Integer, HashMap<Integer, MMatrix>> readFileAsBinaryEncoded(String fileName, int numberOfInputWords, int skipRowsFromStart, HashMap<Integer, String> dictionaryIndexMapping) throws FileNotFoundException, MatrixException {
        StringBuilder readText = readText(fileName, skipRowsFromStart);
        String[] words = readText.toString().split(" ");
        Arrays.setAll(words, index -> words[index].trim());
        Arrays.setAll(words, index -> words[index].replaceAll("[^a-zäöåA-ZÄÖÅ]", "").toLowerCase());
        TreeSet<String> dictionary = new TreeSet<>();
        Collections.addAll(dictionary, words);
        int maxBits = ComputableMatrix.numberOfBits(dictionary.size());
        HashMap<Matrix, Integer> dictionaryBinaryIndexMapping = new HashMap<>();
        HashMap<String, Matrix> dictionaryStringBinaryIndexMapping = new HashMap<>();
        int index = 0;
        int dictionarySize = dictionary.size();
        for (String word : dictionary) {
            dictionaryIndexMapping.put(index, word);
            Matrix binaryMatrix = ComputableMatrix.encodeToBitColumnVector(index, maxBits);
            dictionaryBinaryIndexMapping.put(binaryMatrix, index);
            dictionaryStringBinaryIndexMapping.put(word, binaryMatrix);
            index++;
        }

        ArrayList<Matrix> encodedWords = new ArrayList<>();
        for (String word : words) {
            if (!word.equals("")) {
                encodedWords.add(dictionaryStringBinaryIndexMapping.get(word));
            }
        }

        HashMap<Integer, MMatrix> inputs = new HashMap<>();
        HashMap<Integer, MMatrix> outputs = new HashMap<>();
        HashMap<Integer, HashMap<Integer, MMatrix>> result = new HashMap<>();
        result.put(0, inputs);
        result.put(1, outputs);

        ArrayDeque<Matrix> encodedWordQueue = new ArrayDeque<>();
        LinkedList<Matrix> encodedWordList = new LinkedList<>();
        int pos = 0;
        for (Matrix encodedWord : encodedWords) {
            encodedWordQueue.addLast(encodedWord);
            encodedWordList.addLast(encodedWord);

            if (encodedWordQueue.size() >= numberOfInputWords + 1) {
                ArrayList<Matrix> inputMatrices = new ArrayList<>();
                ArrayList<Matrix> outputMatrices = new ArrayList<>();
                int wordIndex = 0;
                for (Matrix matrix : encodedWordList) {
                    if (wordIndex++ < encodedWordList.size() - 1) inputMatrices.add(matrix);
                    else {
                        Matrix outputMatrix = new DMatrix(dictionarySize, 1);
                        outputMatrix.setValue(dictionaryBinaryIndexMapping.get(matrix), 0, 1);
                        outputMatrices.add(outputMatrix);
                    }
                }

                JMatrix joinedInputMatrix = new JMatrix(inputMatrices, true);
                JMatrix joinedOutputMatrix = new JMatrix(outputMatrices, true);
                inputs.put(pos, new MMatrix(joinedInputMatrix));
                outputs.put(pos, new MMatrix(joinedOutputMatrix));
                pos++;

                encodedWordQueue.removeFirst();
                encodedWordList.removeFirst();
            }
        }

        return result;
    }
/**
     * Reads text file and maps each word as separate binary encoded input and one-hot encoded output.<br>
     *
     * @param fileName name of file to be read.
     * @param numberOfInputWords number of words per input column.
     * @param skipRowsFromStart skips specified number of rows from start.
     * @param dictionaryIndexMapping dictionary index mapping.
     * @return structure containing input and output matrices.
     * @throws FileNotFoundException throws exception if file is not found.
     * @throws MatrixException throws exception if matrix operation fails.
     */
Reads text file and maps each word as separate binary encoded input and one-hot encoded output.
@param fileName name of file to be read.
@param numberOfInputWords number of words per input column.
@param skipRowsFromStart skips specified number of rows from start.
@param dictionaryIndexMapping dictionary index mapping.
@return structure containing input and output matrices.
@throws FileNotFoundException throws exception if file is not found.
@throws MatrixException throws exception if matrix operation fails.
Reads text file and maps each word as separate binary encoded input and one - hot encoded output . @param fileName name of file to be read . @param numberOfInputWords number of words per input column . @param skipRowsFromStart skips specified number of rows from start . @param dictionaryIndexMapping dictionary index mapping . @return structure containing input and output matrices . @throws FileNotFoundException throws exception if file is not found . @throws MatrixException throws exception if matrix operation fails .
public static HashMap<Integer, HashMap<Integer, MMatrix>> readFileAsBinaryEncoded(String fileName, int numberOfInputWords, int skipRowsFromStart, HashMap<Integer, String> dictionaryIndexMapping) throws FileNotFoundException, MatrixException {
        StringBuilder readText = readText(fileName, skipRowsFromStart);
        String[] words = readText.toString().split(" ");
        Arrays.setAll(words, index -> words[index].trim());
        Arrays.setAll(words, index -> words[index].replaceAll("[^a-zäöåA-ZÄÖÅ]", "").toLowerCase());
        TreeSet<String> dictionary = new TreeSet<>();
        Collections.addAll(dictionary, words);
        int maxBits = ComputableMatrix.numberOfBits(dictionary.size());
        HashMap<Matrix, Integer> dictionaryBinaryIndexMapping = new HashMap<>();
        HashMap<String, Matrix> dictionaryStringBinaryIndexMapping = new HashMap<>();
        int index = 0;
        int dictionarySize = dictionary.size();
        for (String word : dictionary) {
            dictionaryIndexMapping.put(index, word);
            Matrix binaryMatrix = ComputableMatrix.encodeToBitColumnVector(index, maxBits);
            dictionaryBinaryIndexMapping.put(binaryMatrix, index);
            dictionaryStringBinaryIndexMapping.put(word, binaryMatrix);
            index++;
        }
        ArrayList<Matrix> encodedWords = new ArrayList<>();
        for (String word : words) {
            if (!word.equals("")) {
                encodedWords.add(dictionaryStringBinaryIndexMapping.get(word));
            }
        }
        HashMap<Integer, MMatrix> inputs = new HashMap<>();
        HashMap<Integer, MMatrix> outputs = new HashMap<>();
        HashMap<Integer, HashMap<Integer, MMatrix>> result = new HashMap<>();
        result.put(0, inputs);
        result.put(1, outputs);
        ArrayDeque<Matrix> encodedWordQueue = new ArrayDeque<>();
        LinkedList<Matrix> encodedWordList = new LinkedList<>();
        int pos = 0;
        for (Matrix encodedWord : encodedWords) {
            encodedWordQueue.addLast(encodedWord);
            encodedWordList.addLast(encodedWord);
            if (encodedWordQueue.size() >= numberOfInputWords + 1) {
                ArrayList<Matrix> inputMatrices = new ArrayList<>();
                ArrayList<Matrix> outputMatrices = new ArrayList<>();
                int wordIndex = 0;
                for (Matrix matrix : encodedWordList) {
                    if (wordIndex++ < encodedWordList.size() - 1) inputMatrices.add(matrix);
                    else {
                        Matrix outputMatrix = new DMatrix(dictionarySize, 1);
                        outputMatrix.setValue(dictionaryBinaryIndexMapping.get(matrix), 0, 1);
                        outputMatrices.add(outputMatrix);
                    }
                }
                JMatrix joinedInputMatrix = new JMatrix(inputMatrices, true);
                JMatrix joinedOutputMatrix = new JMatrix(outputMatrices, true);
                inputs.put(pos, new MMatrix(joinedInputMatrix));
                outputs.put(pos, new MMatrix(joinedOutputMatrix));
                pos++;
                encodedWordQueue.removeFirst();
                encodedWordList.removeFirst();
            }
        }
        return result;
    }
public static HashMap < Integer , HashMap < Integer , MMatrix > > readFileAsBinaryEncoded ( String fileName , int numberOfInputWords , int skipRowsFromStart , HashMap < Integer , String > dictionaryIndexMapping ) throws FileNotFoundException , MatrixException { StringBuilder readText = readText ( fileName , skipRowsFromStart ) ; String [ ] words = readText . toString ( ) . split ( " " ) ; Arrays . setAll ( words , index -> words [ index ] . trim ( ) ) ; Arrays . setAll ( words , index -> words [ index ] . replaceAll ( "[^a-zäöåA-ZÄÖÅ]", ""). t Lo w e rCase()); TreeSet < String > dictionary = new TreeSet < > ( ) ; Collections . addAll ( dictionary , words ) ; int maxBits = ComputableMatrix . numberOfBits ( dictionary . size ( ) ) ; HashMap < Matrix , Integer > dictionaryBinaryIndexMapping = new HashMap < > ( ) ; HashMap < String , Matrix > dictionaryStringBinaryIndexMapping = new HashMap < > ( ) ; int index = 0 ; int dictionarySize = dictionary . size ( ) ; for ( String word : dictionary ) { dictionaryIndexMapping . put ( index , word ) ; Matrix binaryMatrix = ComputableMatrix . encodeToBitColumnVector ( index , maxBits ) ; dictionaryBinaryIndexMapping . put ( binaryMatrix , index ) ; dictionaryStringBinaryIndexMapping . put ( word , binaryMatrix ) ; index ++ ; } ArrayList < Matrix > encodedWords = new ArrayList < > ( ) ; for ( String word : words ) { if ( ! word . equals ( "" ) ) { encodedWords . add ( dictionaryStringBinaryIndexMapping . get ( word ) ) ; } } HashMap < Integer , MMatrix > inputs = new HashMap < > ( ) ; HashMap < Integer , MMatrix > outputs = new HashMap < > ( ) ; HashMap < Integer , HashMap < Integer , MMatrix > > result = new HashMap < > ( ) ; result . put ( 0 , inputs ) ; result . put ( 1 , outputs ) ; ArrayDeque < Matrix > encodedWordQueue = new ArrayDeque < > ( ) ; LinkedList < Matrix > encodedWordList = new LinkedList < > ( ) ; int pos = 0 ; for ( Matrix encodedWord : encodedWords ) { encodedWordQueue . addLast ( encodedWord ) ; encodedWordList . addLast ( encodedWord ) ; if ( encodedWordQueue . size ( ) >= numberOfInputWords + 1 ) { ArrayList < Matrix > inputMatrices = new ArrayList < > ( ) ; ArrayList < Matrix > outputMatrices = new ArrayList < > ( ) ; int wordIndex = 0 ; for ( Matrix matrix : encodedWordList ) { if ( wordIndex ++ < encodedWordList . size ( ) - 1 ) inputMatrices . add ( matrix ) ; else { Matrix outputMatrix = new DMatrix ( dictionarySize , 1 ) ; outputMatrix . setValue ( dictionaryBinaryIndexMapping . get ( matrix ) , 0 , 1 ) ; outputMatrices . add ( outputMatrix ) ; } } JMatrix joinedInputMatrix = new JMatrix ( inputMatrices , true ) ; JMatrix joinedOutputMatrix = new JMatrix ( outputMatrices , true ) ; inputs . put ( pos , new MMatrix ( joinedInputMatrix ) ) ; outputs . put ( pos , new MMatrix ( joinedOutputMatrix ) ) ; pos ++ ; encodedWordQueue . removeFirst ( ) ; encodedWordList . removeFirst ( ) ; } } return result ; }
Reads text file and maps each word as separate binary encoded input and one-hot encoded output.<br>
 @param fileName name of file to be read.
Reads text file and maps each word as separate binary encoded input and one - hot encoded output . <br > @param fileName name of file to be read .