def tokenize_annotated(doc, annotation):
 tokens = tokenize(doc, include_hrefs=False)
 for tok in tokens:
  tok.annotation = annotation
 return tokens

def tokenize_annotated ( doc , annotation ) : tokens = tokenize ( doc , include_hrefs = False ) for tok in tokens : tok . annotation = annotation return tokens
tokenize a document and add an annotation attribute to each token .