<s> '''<STR_LIT>''' <EOL> from __future__ import unicode_literals , absolute_import <EOL> import nltk <EOL> from textblob . taggers import PatternTagger <EOL> from textblob . decorators import requires_nltk_corpus <EOL> from textblob . utils import tree2str , filter_insignificant <EOL> from textblob . base import BaseNPExtractor <EOL> class ChunkParser ( nltk . ChunkParserI ) : <EOL> def __init__ ( self ) : <EOL> self . _trained = False <EOL> @ requires_nltk_corpus <EOL> def train ( self ) : <EOL> '''<STR_LIT>''' <EOL> train_data = [ [ ( t , c ) for _ , t , c in nltk . chunk . tree2conlltags ( sent ) ] <EOL> for sent in <EOL> nltk . corpus . conll2000 . chunked_sents ( '<STR_LIT>' , <EOL> chunk_types = [ '<STR_LIT>' ] ) ] <EOL> unigram_tagger = nltk . UnigramTagger ( train_data ) <EOL> self . tagger = nltk . BigramTagger ( train_data , backoff = unigram_tagger ) <EOL> self . _trained = True <EOL> def parse ( self , sentence ) : <EOL> '''<STR_LIT>''' <EOL> if not self . _trained : <EOL> self . train ( ) <EOL> pos_tags = [ pos for ( word , pos ) in sentence ] <EOL> tagged_pos_tags = self . tagger . tag ( pos_tags ) <EOL> chunktags = [ chunktag for ( pos , chunktag ) in tagged_pos_tags ] <EOL> conlltags = [ ( word , pos , chunktag ) for ( ( word , pos ) , chunktag ) in <EOL> zip ( sentence , chunktags ) ] <EOL> return nltk . chunk . util . conlltags2tree ( conlltags ) <EOL> class ConllExtractor ( BaseNPExtractor ) : <EOL> '''<STR_LIT>''' <EOL> POS_TAGGER = PatternTagger ( ) <EOL> CFG = { <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) : '<STR_LIT>' , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) : '<STR_LIT>' , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) : '<STR_LIT>' , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) : '<STR_LIT>' , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) : '<STR_LIT>' , <EOL> } <EOL> INSIGNIFICANT_SUFFIXES = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> def __init__ ( self , parser = None ) : <EOL> self . parser = ChunkParser ( ) if not parser else parser <EOL> def extract ( self , text ) : <EOL> '''<STR_LIT>''' <EOL> sentences = nltk . tokenize . sent_tokenize ( text ) <EOL> noun_phrases = [ ] <EOL> for sentence in sentences : <EOL> parsed = self . _parse_sentence ( sentence ) <EOL> phrases = [ _normalize_tags ( filter_insignificant ( each , <EOL> self . INSIGNIFICANT_SUFFIXES ) ) for each in parsed <EOL> if isinstance ( each , nltk . tree . Tree ) and each . label ( ) <EOL> == '<STR_LIT>' and len ( filter_insignificant ( each ) ) >= <NUM_LIT:1> <EOL> and _is_match ( each , cfg = self . CFG ) ] <EOL> nps = [ tree2str ( phrase ) for phrase in phrases ] <EOL> noun_phrases . extend ( nps ) <EOL> return noun_phrases <EOL> def _parse_sentence ( self , sentence ) : <EOL> '''<STR_LIT>''' <EOL> tagged = self . POS_TAGGER . tag ( sentence ) <EOL> return self . parser . parse ( tagged ) <EOL> class FastNPExtractor ( BaseNPExtractor ) : <EOL> '''<STR_LIT>''' <EOL> CFG = { <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) : '<STR_LIT>' , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) : '<STR_LIT>' , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) : '<STR_LIT>' , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) : '<STR_LIT>' , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) : '<STR_LIT>' , <EOL> } <EOL> def __init__ ( self ) : <EOL> self . _trained = False <EOL> @ requires_nltk_corpus <EOL> def train ( self ) : <EOL> train_data = nltk . corpus . brown . tagged_sents ( categories = '<STR_LIT>' ) <EOL> regexp_tagger = nltk . RegexpTagger ( [ <EOL> ( r'<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( r'<STR_LIT>' , '<STR_LIT::>' ) , <EOL> ( r'<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( r'<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( r'<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( r'<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( r'<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( r'<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( r'<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( r'<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( r'<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( r'<STR_LIT>' , '<STR_LIT>' ) , <EOL> ] ) <EOL> unigram_tagger = nltk . UnigramTagger ( train_data , backoff = regexp_tagger ) <EOL> self . tagger = nltk . BigramTagger ( train_data , backoff = unigram_tagger ) <EOL> self . _trained = True <EOL> return None <EOL> def _tokenize_sentence ( self , sentence ) : <EOL> '''<STR_LIT>''' <EOL> tokens = nltk . word_tokenize ( sentence ) <EOL> return tokens <EOL> def extract ( self , sentence ) : <EOL> '''<STR_LIT>''' <EOL> if not self . _trained : <EOL> self . train ( ) <EOL> tokens = self . _tokenize_sentence ( sentence ) <EOL> tagged = self . tagger . tag ( tokens ) <EOL> tags = _normalize_tags ( tagged ) <EOL> merge = True <EOL> while merge : <EOL> merge = False <EOL> for x in range ( <NUM_LIT:0> , len ( tags ) - <NUM_LIT:1> ) : <EOL> t1 = tags [ x ] <EOL> t2 = tags [ x + <NUM_LIT:1> ] <EOL> key = t1 [ <NUM_LIT:1> ] , t2 [ <NUM_LIT:1> ] <EOL> value = self . CFG . get ( key , '<STR_LIT>' ) <EOL> if value : <EOL> merge = True <EOL> tags . pop ( x ) <EOL> tags . pop ( x ) <EOL> match = '<STR_LIT>' % ( t1 [ <NUM_LIT:0> ] , t2 [ <NUM_LIT:0> ] ) <EOL> pos = value <EOL> tags . insert ( x , ( match , pos ) ) <EOL> break <EOL> matches = [ t [ <NUM_LIT:0> ] for t in tags if t [ <NUM_LIT:1> ] in [ '<STR_LIT>' , '<STR_LIT>' ] ] <EOL> return matches <EOL> def _normalize_tags ( chunk ) : <EOL> '''<STR_LIT>''' <EOL> ret = [ ] <EOL> for word , tag in chunk : <EOL> if tag == '<STR_LIT>' or tag == '<STR_LIT>' : <EOL> ret . append ( ( word , '<STR_LIT>' ) ) <EOL> continue <EOL> if tag . endswith ( '<STR_LIT>' ) : <EOL> ret . append ( ( word , tag [ : - <NUM_LIT:3> ] ) ) <EOL> continue <EOL> if tag . endswith ( '<STR_LIT:S>' ) : <EOL> ret . append ( ( word , tag [ : - <NUM_LIT:1> ] ) ) <EOL> continue <EOL> ret . append ( ( word , tag ) ) <EOL> return ret <EOL> def _is_match ( tagged_phrase , cfg ) : <EOL> '''<STR_LIT>''' <EOL> copy = list ( tagged_phrase ) <EOL> merge = True <EOL> while merge : <EOL> merge = False <EOL> for i in range ( len ( copy ) - <NUM_LIT:1> ) : <EOL> first , second = copy [ i ] , copy [ i + <NUM_LIT:1> ] <EOL> key = first [ <NUM_LIT:1> ] , second [ <NUM_LIT:1> ] <EOL> value = cfg . get ( key , None ) <EOL> if value : <EOL> merge = True <EOL> copy . pop ( i ) <EOL> copy . pop ( i ) <EOL> match = '<STR_LIT>' . format ( first [ <NUM_LIT:0> ] , second [ <NUM_LIT:0> ] ) <EOL> pos = value <EOL> copy . insert ( i , ( match , pos ) ) <EOL> break <EOL> match = any ( [ t [ <NUM_LIT:1> ] in ( '<STR_LIT>' , '<STR_LIT>' ) for t in copy ] ) <EOL> return match </s>
