<s> from __future__ import division <EOL> from percept . tasks . base import Task <EOL> from percept . tasks . train import Train <EOL> from percept . fields . base import Complex , List , Dict , Float <EOL> from inputs . inputs import SimpsonsFormats <EOL> from percept . utils . models import RegistryCategories , get_namespace <EOL> import logging <EOL> from percept . tests . framework import Tester <EOL> from percept . conf . base import settings <EOL> import re <EOL> from sklearn . cluster import KMeans <EOL> import os <EOL> import json <EOL> from itertools import chain <EOL> from matplotlib import pyplot <EOL> import numpy as np <EOL> from sklearn . decomposition import PCA <EOL> log = logging . getLogger ( __name__ ) <EOL> PUNCTUATION = [ "<STR_LIT:]>" , "<STR_LIT:.>" , "<STR_LIT:!>" , "<STR_LIT:?>" ] <EOL> CHARACTERS = { <EOL> '<STR_LIT>' : [ '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' ] , <EOL> '<STR_LIT>' : [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> '<STR_LIT>' : [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : [ '<STR_LIT>' ] , <EOL> } <EOL> CHARACTER_REPLACEMENT = { <EOL> '<STR_LIT:H>' : "<STR_LIT>" , <EOL> '<STR_LIT:M>' : "<STR_LIT>" , <EOL> "<STR_LIT:L>" : "<STR_LIT>" , <EOL> "<STR_LIT:B>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> } <EOL> class CleanupScriptList ( Task ) : <EOL> data = Complex ( ) <EOL> data_format = SimpsonsFormats . dataframe <EOL> category = RegistryCategories . preprocessors <EOL> namespace = get_namespace ( __module__ ) <EOL> help_text = "<STR_LIT>" <EOL> def train ( self , data , target , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> self . data = self . predict ( data ) <EOL> def predict ( self , data , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> script_removal_values = [ "<STR_LIT>" ] <EOL> for r in script_removal_values : <EOL> data = data [ data [ "<STR_LIT>" ] != r ] <EOL> log . info ( data ) <EOL> data [ '<STR_LIT>' ] = [ i . split ( '<STR_LIT:\n>' ) [ <NUM_LIT:0> ] . strip ( ) for i in data [ '<STR_LIT>' ] ] <EOL> data [ '<STR_LIT>' ] = [ i . split ( '<STR_LIT:/>' ) [ - <NUM_LIT:1> ] . split ( '<STR_LIT>' ) [ <NUM_LIT:0> ] for i in data [ '<STR_LIT:url>' ] ] <EOL> data . index = range ( data . shape [ <NUM_LIT:0> ] ) <EOL> return data <EOL> def check_if_character ( character ) : <EOL> all_characters = list ( chain . from_iterable ( [ CHARACTERS [ k ] for k in CHARACTERS ] ) ) <EOL> return character in all_characters <EOL> def find_replacement ( character ) : <EOL> for k in CHARACTERS : <EOL> if character in CHARACTERS [ k ] : <EOL> return k <EOL> return "<STR_LIT>" <EOL> def cleanup_name ( character ) : <EOL> for k in CHARACTER_REPLACEMENT : <EOL> if k == character : <EOL> character = CHARACTER_REPLACEMENT [ k ] <EOL> return character <EOL> class CleanupScriptText ( Task ) : <EOL> data = Complex ( ) <EOL> data_format = SimpsonsFormats . dataframe <EOL> category = RegistryCategories . preprocessors <EOL> namespace = get_namespace ( __module__ ) <EOL> help_text = "<STR_LIT>" <EOL> def train ( self , data , target , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> self . data = self . predict ( data ) <EOL> def check_for_line_split ( self , line ) : <EOL> return check_if_character ( line . split ( "<STR_LIT::>" ) [ <NUM_LIT:0> ] ) <EOL> def predict ( self , data , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> voice_scripts = [ ] <EOL> for i in xrange ( <NUM_LIT:0> , data . shape [ <NUM_LIT:0> ] ) : <EOL> script_lines = data [ '<STR_LIT>' ] [ i ] . split ( '<STR_LIT:\n>' ) <EOL> voice_lines = [ ] <EOL> current_line = "<STR_LIT>" <EOL> for ( i , line ) in enumerate ( script_lines ) : <EOL> current_line = current_line . strip ( ) <EOL> line = line . strip ( ) <EOL> if line . startswith ( "<STR_LIT:[>" ) and line . endswith ( "<STR_LIT:]>" ) : <EOL> continue <EOL> if line . startswith ( "<STR_LIT:->" ) : <EOL> continue <EOL> voice_line = re . search ( '<STR_LIT>' , line ) <EOL> if voice_line is not None : <EOL> if self . check_for_line_split ( current_line ) : <EOL> voice_lines . append ( current_line ) <EOL> current_line = line <EOL> elif ( len ( line ) == <NUM_LIT:0> or line . startswith ( "<STR_LIT:->" ) ) and len ( current_line ) > <NUM_LIT:0> : <EOL> if self . check_for_line_split ( current_line ) : <EOL> voice_lines . append ( current_line ) <EOL> current_line = "<STR_LIT>" <EOL> voice_lines . append ( "<STR_LIT:U+0020>" ) <EOL> elif len ( current_line ) > <NUM_LIT:0> : <EOL> current_line += "<STR_LIT:U+0020>" + line <EOL> script_text = "<STR_LIT:\n>" . join ( [ l for l in voice_lines if len ( l ) > <NUM_LIT:0> and "<STR_LIT:{>" not in l and "<STR_LIT:=>" not in l ] ) <EOL> script_text = re . sub ( "<STR_LIT>" , "<STR_LIT>" , script_text ) <EOL> voice_scripts . append ( script_text . strip ( ) ) <EOL> data [ '<STR_LIT>' ] = voice_scripts <EOL> return data <EOL> class ReformatScriptText ( Task ) : <EOL> data = Complex ( ) <EOL> voice_lines = Complex ( ) <EOL> data_format = SimpsonsFormats . dataframe <EOL> category = RegistryCategories . preprocessors <EOL> namespace = get_namespace ( __module__ ) <EOL> help_text = "<STR_LIT>" <EOL> args = { '<STR_LIT>' : os . path . abspath ( os . path . join ( settings . PROJECT_PATH , "<STR_LIT>" ) ) , '<STR_LIT>' : True } <EOL> def train ( self , data , target , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> self . data = data <EOL> self . predict ( self . data , ** kwargs ) <EOL> def predict ( self , data , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> voice_scripts = list ( data [ '<STR_LIT>' ] ) <EOL> scriptfile = kwargs [ '<STR_LIT>' ] <EOL> do_replace = kwargs [ '<STR_LIT>' ] <EOL> json_scripts = json . load ( open ( scriptfile ) ) <EOL> voice_scripts += [ s [ '<STR_LIT>' ] for s in json_scripts ] <EOL> script_segments = [ ] <EOL> for script in voice_scripts : <EOL> script = script . replace ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> lines = script . split ( "<STR_LIT:\n>" ) <EOL> segment = [ ] <EOL> for line in lines : <EOL> if line . strip ( ) != "<STR_LIT>" : <EOL> line = line . encode ( '<STR_LIT:ascii>' , '<STR_LIT:ignore>' ) <EOL> line_split = line . split ( "<STR_LIT::>" ) <EOL> if do_replace : <EOL> line_split [ <NUM_LIT:0> ] = find_replacement ( line_split [ <NUM_LIT:0> ] . strip ( ) ) <EOL> line_split [ <NUM_LIT:0> ] = cleanup_name ( line_split [ <NUM_LIT:0> ] . strip ( ) ) <EOL> segment . append ( { '<STR_LIT>' : line_split [ <NUM_LIT:0> ] , <EOL> '<STR_LIT>' : "<STR_LIT::>" . join ( line_split [ <NUM_LIT:1> : ] ) . strip ( ) } ) <EOL> else : <EOL> if len ( segment ) > <NUM_LIT:0> : <EOL> script_segments . append ( segment ) <EOL> segment = [ ] <EOL> if len ( segment ) > <NUM_LIT:0> : <EOL> script_segments . append ( segment ) <EOL> self . voice_lines = script_segments <EOL> class ClusterScriptText ( Task ) : <EOL> data = Complex ( ) <EOL> clusters = Complex ( ) <EOL> predictions = Complex ( ) <EOL> clusters = List ( ) <EOL> cl = Complex ( ) <EOL> vec = Complex ( ) <EOL> vec1 = Complex ( ) <EOL> data_format = SimpsonsFormats . dataframe <EOL> category = RegistryCategories . preprocessors <EOL> namespace = get_namespace ( __module__ ) <EOL> help_text = "<STR_LIT>" <EOL> def train ( self , data , target , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> self . data = data <EOL> self . predict ( self . data ) <EOL> def predict ( self , data , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> from train import Vectorizer , make_df <EOL> self . vec = Vectorizer ( ) <EOL> reformatter = ReformatScriptText ( ) <EOL> args = reformatter . args <EOL> args [ '<STR_LIT>' ] = False <EOL> reformatter . train ( data , "<STR_LIT>" , ** args ) <EOL> script_segments = list ( chain . from_iterable ( reformatter . voice_lines ) ) <EOL> text = [ s [ '<STR_LIT>' ] for s in script_segments ] <EOL> speaker = [ s [ '<STR_LIT>' ] for s in script_segments ] <EOL> unique_speakers = list ( set ( speaker ) ) <EOL> speaker_code_dict = { k : i for ( i , k ) in enumerate ( unique_speakers ) } <EOL> speaker_codes = [ speaker_code_dict [ k ] for k in unique_speakers ] <EOL> speaker_list = [ ] <EOL> speaker_frame = make_df ( [ text , speaker ] , [ "<STR_LIT:text>" , "<STR_LIT>" ] ) <EOL> for i in unique_speakers : <EOL> s_text = "<STR_LIT:\n>" . join ( list ( speaker_frame [ speaker_frame [ '<STR_LIT>' ] == i ] [ '<STR_LIT:text>' ] ) ) <EOL> speaker_list . append ( s_text ) <EOL> self . vec . fit ( speaker_list , speaker_codes , <NUM_LIT:200> , min_features = <NUM_LIT:2> ) <EOL> features = self . vec . batch_get_features ( speaker_list ) <EOL> cl = KMeans ( ) <EOL> self . predictions = cl . fit_predict ( features ) <EOL> self . cl = cl <EOL> for i in xrange ( <NUM_LIT:0> , max ( self . predictions ) ) : <EOL> clust = [ ] <EOL> for c in xrange ( <NUM_LIT:0> , len ( speaker_codes ) ) : <EOL> if self . predictions [ c ] == i : <EOL> clust . append ( unique_speakers [ c ] ) <EOL> self . clusters . append ( clust ) <EOL> pca = PCA ( n_components = <NUM_LIT:2> , whiten = True ) . fit ( features ) <EOL> rf = pca . transform ( features ) <EOL> labels = cl . labels_ <EOL> pyplot . clf ( ) <EOL> centroids = cl . cluster_centers_ <EOL> pyplot . cla ( ) <EOL> for i in range ( max ( labels ) ) : <EOL> ds = rf [ np . where ( labels == i ) ] <EOL> pyplot . plot ( ds [ : , <NUM_LIT:0> ] , ds [ : , <NUM_LIT:1> ] , '<STR_LIT:o>' , label = self . clusters [ i ] [ <NUM_LIT:0> ] ) <EOL> pyplot . legend ( loc = <NUM_LIT:8> ) <EOL> pyplot . savefig ( '<STR_LIT>' ) <EOL> self . vec1 = Vectorizer ( ) <EOL> speaker_codes = [ speaker_code_dict [ k ] for k in speaker ] <EOL> self . vec1 . fit ( text , speaker_codes , <NUM_LIT:200> , min_features = <NUM_LIT:2> ) <EOL> features = self . vec1 . batch_get_features ( text ) <EOL> pca = PCA ( n_components = <NUM_LIT:2> , whiten = True ) . fit ( features ) <EOL> rf = pca . transform ( features ) <EOL> pyplot . clf ( ) <EOL> pyplot . cla ( ) <EOL> for i in range ( len ( speaker_codes ) ) : <EOL> pyplot . plot ( rf [ i , <NUM_LIT:0> ] , rf [ i , <NUM_LIT:1> ] , '<STR_LIT:o>' , label = speaker [ i ] ) <EOL> pyplot . savefig ( '<STR_LIT>' ) <EOL> class CleanupTranscriptList ( CleanupScriptList ) : <EOL> help_text = "<STR_LIT>" <EOL> def predict ( self , data , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> script_removal_values = [ "<STR_LIT>" ] <EOL> for r in script_removal_values : <EOL> data = data [ data [ "<STR_LIT>" ] != r ] <EOL> episode_names = [ ] <EOL> for i in data [ '<STR_LIT:url>' ] : <EOL> match = re . search ( "<STR_LIT>" , i ) <EOL> if match is None : <EOL> episode_names . append ( "<STR_LIT>" ) <EOL> continue <EOL> episode_names . append ( match . group ( <NUM_LIT:0> ) . split ( "<STR_LIT:=>" ) [ <NUM_LIT:1> ] ) <EOL> data [ '<STR_LIT>' ] = episode_names <EOL> data [ '<STR_LIT>' ] = [ int ( i . split ( "<STR_LIT:e>" ) [ <NUM_LIT:0> ] . replace ( "<STR_LIT:s>" , "<STR_LIT>" ) ) for i in episode_names ] <EOL> data [ '<STR_LIT>' ] = [ int ( i . split ( "<STR_LIT:e>" ) [ <NUM_LIT:1> ] ) for i in episode_names ] <EOL> data . index = range ( data . shape [ <NUM_LIT:0> ] ) <EOL> return data <EOL> class CleanupTranscriptText ( CleanupScriptText ) : <EOL> help_text = "<STR_LIT>" <EOL> def predict ( self , data , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> voice_scripts = [ ] <EOL> for i in xrange ( <NUM_LIT:0> , data . shape [ <NUM_LIT:0> ] ) : <EOL> script_lines = data [ '<STR_LIT>' ] [ i ] . split ( '<STR_LIT>' ) <EOL> lines = [ ] <EOL> current_line = "<STR_LIT>" <EOL> for line in script_lines : <EOL> line = line . replace ( "<STR_LIT:->" , "<STR_LIT>" ) <EOL> line = line . replace ( "<STR_LIT:\n>" , "<STR_LIT>" ) <EOL> line = line . replace ( "<STR_LIT:\\>" , "<STR_LIT>" ) <EOL> line = line . strip ( ) <EOL> line = line . encode ( '<STR_LIT:ascii>' , '<STR_LIT:ignore>' ) <EOL> current_line = "<STR_LIT>" . format ( current_line , line ) <EOL> if current_line [ - <NUM_LIT:1> ] in PUNCTUATION : <EOL> lines . append ( current_line ) <EOL> current_line = "<STR_LIT>" <EOL> lines = ( "<STR_LIT:\n>" . join ( lines ) ) <EOL> lines = re . sub ( "<STR_LIT>" , "<STR_LIT>" , lines ) <EOL> lines = re . sub ( "<STR_LIT>" , "<STR_LIT>" , lines ) <EOL> lines = [ l . strip ( ) for l in lines . split ( "<STR_LIT:\n>" ) if len ( l ) > <NUM_LIT:10> ] <EOL> text = "<STR_LIT:\n>" . join ( lines ) <EOL> voice_scripts . append ( text ) <EOL> data [ '<STR_LIT>' ] = voice_scripts <EOL> return data </s>
