given two document graphs which annotate the same text and which use the
    same tokenization, creates a dictionary with a mapping from the token
    IDs used in the first graph to the token IDs used in the second graph.

    Parameters
    ----------
    docgraph_with_old_names : DiscourseDocumentGraph
        a document graph with token IDs that will be replaced later on
    docgraph_with_new_names : DiscourseDocumentGraph
        a document graph with token IDs that will replace the token IDs
        used in ``docgraph_with_old_names`` later on

    Returns
    -------
    old2new : dict
        maps from a token ID used in ``docgraph_with_old_names`` to the token
        ID used in ``docgraph_with_new_names`` to reference the same token

def create_token_mapping(docgraph_with_old_names, docgraph_with_new_names,
                         verbose=False):
    """
    given two document graphs which annotate the same text and which use the
    same tokenization, creates a dictionary with a mapping from the token
    IDs used in the first graph to the token IDs used in the second graph.

    Parameters
    ----------
    docgraph_with_old_names : DiscourseDocumentGraph
        a document graph with token IDs that will be replaced later on
    docgraph_with_new_names : DiscourseDocumentGraph
        a document graph with token IDs that will replace the token IDs
        used in ``docgraph_with_old_names`` later on

    Returns
    -------
    old2new : dict
        maps from a token ID used in ``docgraph_with_old_names`` to the token
        ID used in ``docgraph_with_new_names`` to reference the same token
    """
    def kwic_string(docgraph, keyword_index):
        tokens = [tok for (tokid, tok) in list(docgraph.get_tokens())]
        before, keyword, after = get_kwic(tokens, keyword_index)
        return "{0} (Index: {1}): {2} [[{3}]] {4}\n".format(
            docgraph.name, keyword_index, ' '.join(before), keyword,
            ' '.join(after))

    # generators of (token ID, token) tuples
    old_token_gen = docgraph_with_old_names.get_tokens()
    new_token_gen = docgraph_with_new_names.get_tokens()

    old2new = {}
    for i, (new_tok_id, new_tok) in enumerate(new_token_gen):
        old_tok_id, old_tok = old_token_gen.next()
        if new_tok != old_tok:  # token mismatch
            if verbose:
                raise ValueError(u"Tokenization mismatch:\n{0}{1}".format(
                    kwic_string(docgraph_with_old_names, i),
                    kwic_string(docgraph_with_new_names, i)))
            raise ValueError(
                u"Tokenization mismatch: {0} ({1}) vs. {2} ({3})\n"
                "\t{4} != {5}".format(
                    docgraph_with_new_names.name, docgraph_with_new_names.ns,
                    docgraph_with_old_names.name, docgraph_with_old_names.ns,
                    new_tok, old_tok).encode('utf-8'))
        else:
            old2new[old_tok_id] = new_tok_id
    return old2new