src/Documents/TokensDocument.php
/**
     * Apply the transformation
     * @param ITokenTransformation $transformer
     * @param boolean Remove nulls, we need nulls to indictate where stop words are
     * @return \TextAnalysis\Documents\TokensDocument
     */
Apply the transformation
@param ITokenTransformation $transformer
@param boolean Remove nulls, we need nulls to indictate where stop words are
@return \TextAnalysis\Documents\TokensDocument
Apply the transformation @param ITokenTransformation $transformer @param boolean Remove nulls we need nulls to indictate where stop words are @return \ TextAnalysis \ Documents \ TokensDocument
public function applyTransformation(ITokenTransformation $transformer, $removeNulls = true) 
    {        
        foreach($this->tokens as &$token) 
        { 
            $token = $transformer->transform($token);
        }
        if($removeNulls) {
            $this->tokens = array_values(array_filter($this->tokens));
        }
        return $this;
    }
public function applyTransformation ( ITokenTransformation $ transformer , $ removeNulls = true ) { foreach ( $ this -> tokens as & $ token ) { $ token = $ transformer -> transform ( $ token ) ; } if ( $ removeNulls ) { $ this -> tokens = array_values ( array_filter ( $ this -> tokens ) ) ; } return $ this ; }
Apply the transformation
 @param ITokenTransformation $transformer
 @param boolean Remove nulls, we need nulls to indictate where stop words are
 @return \TextAnalysis\Documents\TokensDocument
Apply the transformation @param ITokenTransformation $transformer @param boolean Remove nulls we need nulls to indictate where stop words are @return \ TextAnalysis \ Documents \ TokensDocument
//filter null tokens and re-index