src/main/java/pt/it/av/tnav/ml/tm/TM.java
public static Pipeline DPPipeline(NGram term, Tokenizer t, StopWords sw, int min, int max,
                                      int n, int nn, Stemmer s, Map<NGram, MutableDouble> m) {
        Pipeline pipeline = new Pipeline();
        CircularQueue<String> q = new CircularQueue<>(n * 2 + term.size());
        NGram stemm = s.stem(term);
        String array[] = new String[term.size()];

        // Setences
        pipeline.addLast((Object o, List<Object> l) -> {
            String input = (String) o;
            Iterator<String> setences = StringUtils.splitSetences(input);
            while (setences.hasNext())
                l.add(setences.next());
        });

        // Clean the text  and convert it to tokens
        pipeline.addLast((Object o, List<Object> l) -> {
            String setence = (String) o;
            Iterator<String> tokens = t.tokenizeIt(setence);
            while (tokens.hasNext())
                l.add(tokens.next());
        });

        // Remove stop words and words that are too small or too big
        pipeline.addLast((Object o, List<Object> l) -> {
            String token = (String) o;
            if (!sw.isStopWord(token)) {
                if (token.length() >= min && token.length() <= max)
                    l.add(token);
            }
        });

        // Extract the neighborhoods of the token
        pipeline.addLast((Object o, List<Object> l) -> {
            String token = (String) o;
            q.add(token);
            if (q.isFull()) {
                q.middle(array);
                //System.err.print("Array: "+ PrintUtils.array(array));
                if (stemm.equals(s.stem(array))) {
                    //System.err.print(" -> Match");
                    String[] buffer = q.toArray(new String[n * 2 + term.size()]);
                    l.add(TM.extractNGrams(buffer, 0, n, nn));
                    //System.err.print(" "+PrintUtils.list(TM.extractNGrams(buffer, 0, n, nn)));
                    l.add(TM.extractNGrams(buffer, n + term.size(), n * 2 + term.size(), nn));
                    //System.err.println(" "+PrintUtils.list(TM.extractNGrams(buffer, n + term.size(), n * 2 + term.size(), nn)));
                }
            }
        });

        // Count Raw Frequency
        pipeline.addLast((Object o, List<Object> l) -> {
            List<NGram> candidates = Utils.cast(o);
            for (NGram ngram : candidates) {
                if (!m.containsKey(ngram))
                    m.put(ngram, new MutableDouble());
                m.get(ngram).increment();
            }
        });

        return pipeline;
    }
/**
     * Returns a Pipeline ready to process text and extract the neighborhood of some term.
     *
     * @param term the term used to extract the neighborhood
     * @param t    tokenizer to convert text into valid tokens
     * @param sw   stop words filter, used to remove unwanted tokens
     * @param min  minimal size of token
     * @param max  maximal size of token
     * @param n    size of the neighborhood
     * @param nn   size of the extracted n-grams
     * @param s    Stemmer used to match tokens with the term
     * @param m    Map used to store the term's neighborhood
     * @return a Pipeline ready to process text and find the neighborhood.
     * @see pt.it.av.atnog.utils.parallel.Pipeline
     * @see Tokenizer
     * @see StopWords
     * @see Stemmer
     * @see java.util.Map
     */
Returns a Pipeline ready to process text and extract the neighborhood of some term.
Returns a Pipeline ready to process text and extract the neighborhood of some term .
public static Pipeline DPPipeline(NGram term, Tokenizer t, StopWords sw, int min, int max,
                                      int n, int nn, Stemmer s, Map<NGram, MutableDouble> m) {
        Pipeline pipeline = new Pipeline();
        CircularQueue<String> q = new CircularQueue<>(n * 2 + term.size());
        NGram stemm = s.stem(term);
        String array[] = new String[term.size()];
        pipeline.addLast((Object o, List<Object> l) -> {
            String input = (String) o;
            Iterator<String> setences = StringUtils.splitSetences(input);
            while (setences.hasNext())
                l.add(setences.next());
        });
        pipeline.addLast((Object o, List<Object> l) -> {
            String setence = (String) o;
            Iterator<String> tokens = t.tokenizeIt(setence);
            while (tokens.hasNext())
                l.add(tokens.next());
        });
        pipeline.addLast((Object o, List<Object> l) -> {
            String token = (String) o;
            if (!sw.isStopWord(token)) {
                if (token.length() >= min && token.length() <= max)
                    l.add(token);
            }
        });
        pipeline.addLast((Object o, List<Object> l) -> {
            String token = (String) o;
            q.add(token);
            if (q.isFull()) {
                q.middle(array);
                if (stemm.equals(s.stem(array))) {
                    String[] buffer = q.toArray(new String[n * 2 + term.size()]);
                    l.add(TM.extractNGrams(buffer, 0, n, nn));
                    l.add(TM.extractNGrams(buffer, n + term.size(), n * 2 + term.size(), nn));
                }
            }
        });
        pipeline.addLast((Object o, List<Object> l) -> {
            List<NGram> candidates = Utils.cast(o);
            for (NGram ngram : candidates) {
                if (!m.containsKey(ngram))
                    m.put(ngram, new MutableDouble());
                m.get(ngram).increment();
            }
        });
        return pipeline;
    }
public static Pipeline DPPipeline ( NGram term , Tokenizer t , StopWords sw , int min , int max , int n , int nn , Stemmer s , Map < NGram , MutableDouble > m ) { Pipeline pipeline = new Pipeline ( ) ; CircularQueue < String > q = new CircularQueue < > ( n * 2 + term . size ( ) ) ; NGram stemm = s . stem ( term ) ; String array [ ] = new String [ term . size ( ) ] ; pipeline . addLast ( ( Object o , List < Object > l ) -> { String input = ( String ) o ; Iterator < String > setences = StringUtils . splitSetences ( input ) ; while ( setences . hasNext ( ) ) l . add ( setences . next ( ) ) ; } ) ; pipeline . addLast ( ( Object o , List < Object > l ) -> { String setence = ( String ) o ; Iterator < String > tokens = t . tokenizeIt ( setence ) ; while ( tokens . hasNext ( ) ) l . add ( tokens . next ( ) ) ; } ) ; pipeline . addLast ( ( Object o , List < Object > l ) -> { String token = ( String ) o ; if ( ! sw . isStopWord ( token ) ) { if ( token . length ( ) >= min && token . length ( ) <= max ) l . add ( token ) ; } } ) ; pipeline . addLast ( ( Object o , List < Object > l ) -> { String token = ( String ) o ; q . add ( token ) ; if ( q . isFull ( ) ) { q . middle ( array ) ; if ( stemm . equals ( s . stem ( array ) ) ) { String [ ] buffer = q . toArray ( new String [ n * 2 + term . size ( ) ] ) ; l . add ( TM . extractNGrams ( buffer , 0 , n , nn ) ) ; l . add ( TM . extractNGrams ( buffer , n + term . size ( ) , n * 2 + term . size ( ) , nn ) ) ; } } } ) ; pipeline . addLast ( ( Object o , List < Object > l ) -> { List < NGram > candidates = Utils . cast ( o ) ; for ( NGram ngram : candidates ) { if ( ! m . containsKey ( ngram ) ) m . put ( ngram , new MutableDouble ( ) ) ; m . get ( ngram ) . increment ( ) ; } } ) ; return pipeline ; }
Returns a Pipeline ready to process text and extract the neighborhood of some term.
Returns a Pipeline ready to process text and extract the neighborhood of some term .
// Setences // Clean the text  and convert it to tokens // Remove stop words and words that are too small or too big // Extract the neighborhoods of the token //System.err.print("Array: "+ PrintUtils.array(array)); //System.err.print(" -> Match"); //System.err.print(" "+PrintUtils.list(TM.extractNGrams(buffer, 0, n, nn))); //System.err.println(" "+PrintUtils.list(TM.extractNGrams(buffer, n + term.size(), n * 2 + term.size(), nn))); // Count Raw Frequency