import glob,os,sys

class Path():
    '''
    >>> paths = Path(source,"*.txt")
    >>> for path in paths:
            lines = Stream(path)
                for line in lines:
                    print(line)
    '''
    
    def __init__(self, source, pattern):
        self.source = source
        self.pattern = pattern     
  
    def __getpaths__(self):
        source = os.path.join(self.source, self.pattern)
        
        files = glob.glob(source)
        for filename in files:
            yield os.path.join(source, filename)
                
    def __iter__(self):
        return self.__getpaths__()         
                

class Stream():
    '''
    >>> lines = Stream(path)
    >>> for line in lines:
            print(line)
    '''        
    def __init__(self, 
            encoding=None,
            sentencizer=None,
            text_filters=[]
           ):
        
        self.encoding = encoding
        self.__sentencizer = sentencizer
        self.__text_filters = text_filters
        
    def __call__(self,path):
        """Read lines from filepath."""
        
        
        with open(path,'r',
            encoding = (
                self.encoding(path)  
                if callable(self.encoding) 
                else self.encoding)  
            ) as fd:
            # обрабатываем либо по предложению    
            if self.__sentencizer:
                text = self.preprocess_text(fd.read())
                for sentence in self.__sentencizer(text):
                    yield sentence
            # либо по строке
            else:  
                for line in fd:
                    yield line
    
    def preprocess_text(self,text):
        for text_filter in self.__text_filters:
            text = text_filter(text)
        return text
    

class Lemmatizer():
    
    def __init__(self, lemmatizer=None,
                allowed_tags=set(), disallowed_tags=set()):
        self.lemmatize = lemmatizer
        self.allowed_tags = set(allowed_tags) - set(disallowed_tags)
        
        
    def __call__(self,data):
        
        if isinstance(data,(str)):
            data = [data]   
        self.allowed_tags 
        
        for lemma,pos in self.lemmatize(data,pos=True):
            if self.allowed_tags:
                if (self.allowed_tags) and (pos in self.allowed_tags):
                    yield lemma   
            else:
                 yield lemma
            


class Tokenizer():
    
    def __init__(self,tokenizer=None):
        self.tokenize = tokenizer
        
    def __call__(self,data):
        return self.tokenize(data)    
  

class CharCleaner():
    
    def __init__(self,cleaners=None):
        self.cleaners = cleaners
        
    def __call__(self,data):
        for cleaner in self.cleaners:
            data = cleaner(data)
        
        return data  


class TokenCleaner():
    
    def __init__(self,cleaners=None):
        self.cleaners = cleaners
        
    def __call__(self,data):
        for cleaner in self.cleaners:
            data = cleaner(data)
        
        return data  


class LemmaCleaner():
    
    def __init__(self,cleaners=None):
        self.cleaners = cleaners
        
    def __call__(self,data):
        for cleaner in self.cleaners:
            data = cleaner(data)
        
        return data  

import glob,os,sys

class Path():
    '''
    >>> paths = Path(source,"*.txt")
    >>> for path in paths:
            lines = Stream(path)
                for line in lines:
                    print(line)
    '''
    
    def __init__(self, source, pattern):
        self.source = source
        self.pattern = pattern     
  
    def __getpaths__(self):
        source = os.path.join(self.source, self.pattern)
        
        files = glob.glob(source)
        for filename in files:
            yield os.path.join(source, filename)
                
    def __iter__(self):
        return self.__getpaths__()         
                

class Stream():
    '''
    >>> lines = Stream(path)
    >>> for line in lines:
            print(line)
    '''        
    def __init__(self, 
            encoding=None,
            sentencizer=None,
            text_filters=[]
           ):
        
        self.encoding = encoding
        self.__sentencizer = sentencizer
        self.__text_filters = text_filters
        
    def __call__(self,path):
        """Read lines from filepath."""
        
        
        with open(path,'r',
            encoding = (
                self.encoding(path)  
                if callable(self.encoding) 
                else self.encoding)  
            ) as fd:
            # обрабатываем либо по предложению    
            if self.__sentencizer:
                text = self.preprocess_text(fd.read())
                for sentence in self.__sentencizer(text):
                    yield sentence
            # либо по строке
            else:  
                for line in fd:
                    yield line
    
    def preprocess_text(self,text):
        for text_filter in self.__text_filters:
            text = text_filter(text)
        return text
    

class Lemmatizer():
    
    def __init__(self, lemmatizer=None,
                allowed_tags=set(), disallowed_tags=set()):
        self.lemmatize = lemmatizer
        self.allowed_tags = set(allowed_tags) - set(disallowed_tags)
        
        
    def __call__(self,data):
        
        if isinstance(data,(str)):
            data = [data]   
        self.allowed_tags 
        
        for lemma,pos in self.lemmatize(data,pos=True):
            if self.allowed_tags:
                if (self.allowed_tags) and (pos in self.allowed_tags):
                    yield lemma   
            else:
                 yield lemma
            


class Tokenizer():
    
    def __init__(self,tokenizer=None):
        self.tokenize = tokenizer
        
    def __call__(self,data):
        return self.tokenize(data)    
  

class CharCleaner():
    
    def __init__(self,cleaners=None):
        self.cleaners = cleaners
        
    def __call__(self,data):
        for cleaner in self.cleaners:
            data = cleaner(data)
        
        return data  


class TokenCleaner():
    
    def __init__(self,cleaners=None):
        self.cleaners = cleaners
        
    def __call__(self,data):
        for cleaner in self.cleaners:
            data = cleaner(data)
        
        return data  


class LemmaCleaner():
    
    def __init__(self,cleaners=None):
        self.cleaners = cleaners
        
    def __call__(self,data):
        for cleaner in self.cleaners:
            data = cleaner(data)
        
        return data  

>>> paths = Path(source,"*.txt")     >>> for path in paths:             lines = Stream(path)                 for line in lines:                     print(line) >>> lines = Stream(path)     >>> for line in lines:             print(line) Read lines from filepath. # обрабатываем либо по предложению # либо по строке