def clean(self):
        """ Removes any Units that are not applicable given the current semantic or phonetic category.

        Modifies:
            - self.unit_list: Removes Units from this list that do not fit into the clustering category.
                it does by by either combining units to make compound words, combining units with the
                same stem, or eliminating units altogether if they do not conform to the category.
                 If the type is phonetic, this method also generates phonetic clusters for all Unit
                 objects in self.unit_list.


        This method performs three main tasks:
            1. Removes words that do not conform to the clustering category (i.e. start with the
                wrong letter, or are not an animal).
            2. Combine adjacent words with the same stem into a single unit. The NLTK Porter Stemmer
                is used for determining whether stems are the same.
                http://www.nltk.org/_modules/nltk/stem/porter.html
            3. In the case of PHONETIC clustering, compute the phonetic representation of each unit.

        """

        if not self.quiet:
            print
            print "Preprocessing input..."
            print "Raw response:"
            print self.display()

        if not self.quiet:
            print
            print "Cleaning words..."

        #weed out words not starting with the right letter or in the right category
        current_index = 0
        while current_index < len(self.unit_list):
            word = self.unit_list[current_index].text
            if self.type == "PHONETIC":
                test = (word.startswith(self.letter_or_category) and  #starts with required letter
                        not word.endswith('-') and  # Weed out word fragments
                            '_' not in word and # Weed out, e.g., 'filledpause_um'
                        word.lower() in self.english_words) #make sure the word is english
            elif self.type == "SEMANTIC":
                test = word in self.permissible_words
            if not test: #if test fails remove word
                self.remove_unit(index = current_index)
            else: # otherwise just increment, but check to see if you're at the end of the list
                current_index += 1

        #combine words with the same stem
        current_index = 0
        finished = False
        while current_index < len(self.unit_list) - 1:
            #don't combine for lists of length 0, 1
            if stemmer.stem(self.unit_list[current_index].text) == \
                stemmer.stem(self.unit_list[current_index + 1].text):
                #if same stem as next, merge next unit with current unit
                self.combine_same_stem_units(index = current_index)
            else: # if not same stem, increment index
                current_index += 1

        #get phonetic representations
        if self.type == "PHONETIC":
            for unit in self.unit_list:
                word = unit.text

                #get phonetic representation
                if word in self.cmudict:
                    # If word in CMUdict, get its phonetic representation
                    phonetic_representation = self.cmudict[word]
                if word not in self.cmudict:
                    # Else, generate a phonetic representation for it
                    phonetic_representation = self.generate_phonetic_representation(word)
                    phonetic_representation = self.modify_phonetic_representation(phonetic_representation)

                unit.phonetic_representation = phonetic_representation

        if not self.quiet:
            print
            print "Cleaned response:"
            print self.display()
def clean ( self ) : if not self . quiet : print print "Preprocessing input..." print "Raw response:" print self . display ( ) if not self . quiet : print print "Cleaning words..." current_index = 0 while current_index < len ( self . unit_list ) : word = self . unit_list [ current_index ] . text if self . type == "PHONETIC" : test = ( word . startswith ( self . letter_or_category ) and not word . endswith ( '-' ) and '_' not in word and word . lower ( ) in self . english_words ) elif self . type == "SEMANTIC" : test = word in self . permissible_words if not test : self . remove_unit ( index = current_index ) else : current_index += 1 current_index = 0 finished = False while current_index < len ( self . unit_list ) - 1 : if stemmer . stem ( self . unit_list [ current_index ] . text ) == stemmer . stem ( self . unit_list [ current_index + 1 ] . text ) : self . combine_same_stem_units ( index = current_index ) else : current_index += 1 if self . type == "PHONETIC" : for unit in self . unit_list : word = unit . text if word in self . cmudict : phonetic_representation = self . cmudict [ word ] if word not in self . cmudict : phonetic_representation = self . generate_phonetic_representation ( word ) phonetic_representation = self . modify_phonetic_representation ( phonetic_representation ) unit . phonetic_representation = phonetic_representation if not self . quiet : print print "Cleaned response:" print self . display ( )
Removes any Units that are not applicable given the current semantic or phonetic category.

        Modifies:
            - self.unit_list: Removes Units from this list that do not fit into the clustering category.
                it does by by either combining units to make compound words, combining units with the
                same stem, or eliminating units altogether if they do not conform to the category.
                 If the type is phonetic, this method also generates phonetic clusters for all Unit
                 objects in self.unit_list.


        This method performs three main tasks:
            1. Removes words that do not conform to the clustering category (i.e. start with the
                wrong letter, or are not an animal).
            2. Combine adjacent words with the same stem into a single unit. The NLTK Porter Stemmer
                is used for determining whether stems are the same.
                http://www.nltk.org/_modules/nltk/stem/porter.html
            3. In the case of PHONETIC clustering, compute the phonetic representation of each unit.