original_docstring: Return a list of word-token-lists, one per document.
  	Words are optionally lemmatized with WordNet.
docstring: Return a list of word-token-lists, one per document.
  Words are optionally lemmatized with WordNet.
docstring_tokens: Return a list of word - token - lists one per document . Words are optionally lemmatized with WordNet .
code: def load_docs(direc, lemmatize, labelMapFile='labels.csv'):
  	"""Return a list of word-token-lists, one per document.
  	Words are optionally lemmatized with WordNet."""
  
  	labelMap = {}   # docID => gold label, loaded from mapping file
  	with open(os.path.join(direc, labelMapFile)) as inF:
  		for ln in inF:
  			docid, label = ln.strip().split(',')
  			assert docid not in labelMap
  			labelMap[docid] = label
  
  	# create parallel lists of documents and labels
  	docs, labels = [], []
  	for file_path in glob.glob(os.path.join(direc, '*.txt')):
  		filename = os.path.basename(file_path)
  		# open the file at file_path, construct a list of its word tokens,
  		# and append that list to 'docs'.
  		# look up the document's label and append it to 'labels'.
  		f = open(file_path, 'r')
  		
  		word_tokens = []
  
  		# unigram
  		word_tokens += f.read().strip().split()
  
  		# bigram
  		for line in f:
  			words = line.split()
  			if words:
  				previous_word = '<s>'
  				for w in words:
  					word_tokens.append(previous_word + ' ' + w)
  					previous_word = w
  
  		# trigram
  		for line in f:
  			words = line.split()
  			if words:
  				pp_word = '<s>'
  				p_word = '<s>'
  				for w in words:
  					word_tokens.append(pp_word + ' ' + p_word + ' ' + w)
  					pp_word = p_word
  					p_word = w
  
  		# character unigram
  		# word_tokens += list(f.read().replace(' ', '').replace('\n', ''))
  		
  		# character bigram
  		# for line in f:
  		# 	words = line.split()
  		# 	if words:
  		# 		for w in words:
  		# 			p_char = '<s>'
  		# 			for c in w:
  		# 				word_tokens.append(p_char+c)
  		# 				p_char = c
  
  		# lowercase normalization
  		# word_tokens = [w.lower() for w in word_tokens]
  
  		# word length useless
  		# temp = f.read().strip().split()
  		# word_tokens += [len(t) for t in temp]
  		
  		# sentence length useless
  		# for line in f:
  		# 	words = line.split()
  		# 	if words:
  		# 		word_tokens.append(len(words))
  
  		# debug
  		# print (word_tokens)
  
  		docs.append(word_tokens)
  		labels.append(labelMap[filename])
  		f.close()
  
  	return docs, labels
code_tokens: def load_docs ( direc , lemmatize , labelMapFile = 'labels.csv' ) : labelMap = { } with open ( os . path . join ( direc , labelMapFile ) ) as inF : for ln in inF : docid , label = ln . strip ( ) . split ( ',' ) assert docid not in labelMap labelMap [ docid ] = label docs , labels = [ ] , [ ] for file_path in glob . glob ( os . path . join ( direc , '*.txt' ) ) : filename = os . path . basename ( file_path ) f = open ( file_path , 'r' ) word_tokens = [ ] word_tokens += f . read ( ) . strip ( ) . split ( ) for line in f : words = line . split ( ) if words : previous_word = '<s>' for w in words : word_tokens . append ( previous_word + ' ' + w ) previous_word = w for line in f : words = line . split ( ) if words : pp_word = '<s>' p_word = '<s>' for w in words : word_tokens . append ( pp_word + ' ' + p_word + ' ' + w ) pp_word = p_word p_word = w docs . append ( word_tokens ) labels . append ( labelMap [ filename ] ) f . close ( ) return docs , labels
short_docstring: Return a list of word-token-lists, one per document.
short_docstring_tokens: Return a list of word - token - lists one per document .
comment: """Return a list of word-token-lists, one per document.
  	Words are optionally lemmatized with WordNet."""$SEP$# docID => gold label, loaded from mapping file$SEP$# create parallel lists of documents and labels$SEP$# open the file at file_path, construct a list of its word tokens,$SEP$# and append that list to 'docs'.$SEP$# look up the document's label and append it to 'labels'.$SEP$# unigram$SEP$# bigram$SEP$# trigram$SEP$# character unigram$SEP$# word_tokens += list(f.read().replace(' ', '').replace('\n', ''))$SEP$# character bigram$SEP$# for line in f:$SEP$# 	words = line.split()$SEP$# 	if words:$SEP$# 		for w in words:$SEP$# 			p_char = '<s>'$SEP$# 			for c in w:$SEP$# 				word_tokens.append(p_char+c)$SEP$# 				p_char = c$SEP$# lowercase normalization$SEP$# word_tokens = [w.lower() for w in word_tokens]$SEP$# word length useless$SEP$# temp = f.read().strip().split()$SEP$# word_tokens += [len(t) for t in temp]$SEP$# sentence length useless$SEP$# for line in f:$SEP$# 	words = line.split()$SEP$# 	if words:$SEP$# 		word_tokens.append(len(words))$SEP$# debug$SEP$# print (word_tokens)