<s> """<STR_LIT>""" <EOL> import sys <EOL> import csv <EOL> import re <EOL> from textblob . nltk . tokenize import RegexpTokenizer <EOL> from textblob . nltk . stem import PorterStemmer <EOL> tokenizer = RegexpTokenizer ( r'<STR_LIT>' ) <EOL> stemmer = PorterStemmer ( ) <EOL> writer = csv . writer ( sys . stdout ) <EOL> with open ( sys . argv [ <NUM_LIT:1> ] , '<STR_LIT:r>' ) as stream : <EOL> reader = csv . reader ( stream ) <EOL> for document_id , text in reader : <EOL> words = [ ] <EOL> for token in tokenizer . tokenize ( text ) : <EOL> words . append ( stemmer . stem ( token . lower ( ) ) ) <EOL> writer . writerow ( [ document_id , '<STR_LIT:U+0020>' . join ( words ) ] ) </s>
