def build_extraction_dataset(folder, dataset_filename,
                             sender_known=True):
    """Builds signature extraction dataset using emails in the `folder`.

    The emails in the `folder` should be annotated i.e. signature lines
    should be marked with `#sig#`.
    """
    if os.path.exists(dataset_filename):
        os.remove(dataset_filename)
    with open(dataset_filename, 'a') as dataset:
        for filename in os.listdir(folder):
            filename = os.path.join(folder, filename)
            sender, msg = parse_msg_sender(filename, sender_known)
            if not sender or not msg:
                continue
            lines = msg.splitlines()
            for i in range(1, min(SIGNATURE_MAX_LINES,
                                  len(lines)) + 1):
                line = lines[-i]
                label = -1
                if line[:len(SIGNATURE_ANNOTATION)] == \
                        SIGNATURE_ANNOTATION:
                    label = 1
                    line = line[len(SIGNATURE_ANNOTATION):]
                elif line[:len(REPLY_ANNOTATION)] == REPLY_ANNOTATION:
                    line = line[len(REPLY_ANNOTATION):]

                X = build_pattern(line, features(sender))
                X.append(label)
                labeled_pattern = ','.join([str(e) for e in X])
                dataset.write(labeled_pattern + '\n')
def build_extraction_dataset ( folder , dataset_filename , sender_known = True ) : if os . path . exists ( dataset_filename ) : os . remove ( dataset_filename ) with open ( dataset_filename , 'a' ) as dataset : for filename in os . listdir ( folder ) : filename = os . path . join ( folder , filename ) sender , msg = parse_msg_sender ( filename , sender_known ) if not sender or not msg : continue lines = msg . splitlines ( ) for i in range ( 1 , min ( SIGNATURE_MAX_LINES , len ( lines ) ) + 1 ) : line = lines [ - i ] label = - 1 if line [ : len ( SIGNATURE_ANNOTATION ) ] == SIGNATURE_ANNOTATION : label = 1 line = line [ len ( SIGNATURE_ANNOTATION ) : ] elif line [ : len ( REPLY_ANNOTATION ) ] == REPLY_ANNOTATION : line = line [ len ( REPLY_ANNOTATION ) : ] X = build_pattern ( line , features ( sender ) ) X . append ( label ) labeled_pattern = ',' . join ( [ str ( e ) for e in X ] ) dataset . write ( labeled_pattern + '\n' )
Builds signature extraction dataset using emails in the `folder`.

    The emails in the `folder` should be annotated i.e. signature lines
    should be marked with `#sig#`.