Given a tweet python generator, we encode the information into mention and retweet graphs.

    We assume that the tweets are given in increasing timestamp.

    Inputs:  - tweet_generator: A python generator of tweets in python dictionary (json) format.

    Outputs: - mention_graph: The mention graph as a SciPy sparse matrix.
             - user_id_set: A python set containing the Twitter ids for all the dataset users.
             - node_to_id: A python dictionary that maps from node anonymized ids, to twitter user ids.

def extract_graphs_from_tweets(tweet_generator):
    """
    Given a tweet python generator, we encode the information into mention and retweet graphs.

    We assume that the tweets are given in increasing timestamp.

    Inputs:  - tweet_generator: A python generator of tweets in python dictionary (json) format.

    Outputs: - mention_graph: The mention graph as a SciPy sparse matrix.
             - user_id_set: A python set containing the Twitter ids for all the dataset users.
             - node_to_id: A python dictionary that maps from node anonymized ids, to twitter user ids.
    """
    ####################################################################################################################
    # Prepare for iterating over tweets.
    ####################################################################################################################
    # These are initialized as lists for incremental extension.
    tweet_id_set = set()
    user_id_set = list()
    twitter_to_reveal_user_id = dict()

    add_tweet_id = tweet_id_set.add
    append_user_id = user_id_set.append

    # Initialize sparse matrix arrays.
    mention_graph_row = list()
    mention_graph_col = list()

    retweet_graph_row = list()
    retweet_graph_col = list()

    append_mention_graph_row = mention_graph_row.append
    append_mention_graph_col = mention_graph_col.append

    append_retweet_graph_row = retweet_graph_row.append
    append_retweet_graph_col = retweet_graph_col.append

    # Initialize dictionaries.
    id_to_node = dict()
    id_to_name = dict()
    id_to_username = dict()
    id_to_listedcount = dict()

    ####################################################################################################################
    # Iterate over tweets.
    ####################################################################################################################
    counter = 0
    for tweet in tweet_generator:
        # Increment tweet counter.
        counter += 1
        if counter % 10000 == 0:
            print(counter)
        # print(counter)

        # Extract base tweet's values.
        try:
            tweet_id = tweet["id"]
            user_id = tweet["user"]["id"]
            user_screen_name = tweet["user"]["screen_name"]
            user_name = tweet["user"]["name"]

            listed_count_raw = tweet["user"]["listed_count"]

            tweet_in_reply_to_user_id = tweet["in_reply_to_user_id"]
            tweet_in_reply_to_screen_name = tweet["in_reply_to_screen_name"]
            tweet_entities_user_mentions = tweet["entities"]["user_mentions"]
        except KeyError:
            continue

        # Map users to distinct integer numbers.
        graph_size = len(id_to_node)
        source_node = id_to_node.setdefault(user_id, graph_size)

        if listed_count_raw is None:
            id_to_listedcount[user_id] = 0
        else:
            id_to_listedcount[user_id] = int(listed_count_raw)

        # Update sets, lists and dictionaries.
        add_tweet_id(tweet_id)
        id_to_name[user_id] = user_screen_name
        id_to_username[user_id] = user_name
        append_user_id(user_id)

        # twitter_to_user_id

        ################################################################################################################
        # We are dealing with an original tweet.
        ################################################################################################################
        if "retweeted_status" not in tweet.keys():
            ############################################################################################################
            # Update mention matrix.
            ############################################################################################################
            # Get mentioned user ids.
            mentioned_user_id_set = list()
            if tweet_in_reply_to_user_id is not None:
                mentioned_user_id_set.append(tweet_in_reply_to_user_id)

                id_to_name[tweet_in_reply_to_user_id] = tweet_in_reply_to_screen_name
            for user_mention in tweet_entities_user_mentions:
                mentioned_user_id = user_mention["id"]  # TODO: Perhaps safe extract as well.
                mentioned_user_id_set.append(mentioned_user_id)

                id_to_name[mentioned_user_id] = user_mention["screen_name"]  # TODO: Perhaps safe extract as well.

            # We remove duplicates.
            mentioned_user_id_set = set(mentioned_user_id_set)

            # Update the mention graph one-by-one.
            for mentioned_user_id in mentioned_user_id_set:
                # Map users to distinct integer numbers.
                graph_size = len(id_to_node)
                mention_target_node = id_to_node.setdefault(mentioned_user_id, graph_size)

                append_user_id(mentioned_user_id)

                # Add values to the sparse matrix arrays.
                append_mention_graph_row(source_node)
                append_mention_graph_col(mention_target_node)

        ################################################################################################################
        # We are dealing with a retweet.
        ################################################################################################################
        else:
            # Extract base tweet's values.
            original_tweet = tweet["retweeted_status"]

            try:
                original_tweet_id = original_tweet["id"]
                original_tweet_user_id = original_tweet["user"]["id"]
                original_tweet_user_screen_name = original_tweet["user"]["screen_name"]
                original_tweet_user_name = original_tweet["user"]["name"]

                listed_count_raw = original_tweet["user"]["listed_count"]

                original_tweet_in_reply_to_user_id = original_tweet["in_reply_to_user_id"]
                original_tweet_in_reply_to_screen_name = original_tweet["in_reply_to_screen_name"]
                original_tweet_entities_user_mentions = original_tweet["entities"]["user_mentions"]
            except KeyError:
                continue

            # Map users to distinct integer numbers.
            graph_size = len(id_to_node)
            original_tweet_node = id_to_node.setdefault(original_tweet_user_id, graph_size)

            if listed_count_raw is None:
                id_to_listedcount[original_tweet_user_id] = 0
            else:
                id_to_listedcount[original_tweet_user_id] = int(listed_count_raw)

            # Update retweet graph.
            append_retweet_graph_row(source_node)
            append_retweet_graph_col(original_tweet_node)

            # Get mentioned user ids.
            mentioned_user_id_set = list()
            if original_tweet_in_reply_to_user_id is not None:
                mentioned_user_id_set.append(original_tweet_in_reply_to_user_id)

                id_to_name[original_tweet_in_reply_to_user_id] = original_tweet_in_reply_to_screen_name
            for user_mention in original_tweet_entities_user_mentions:
                mentioned_user_id = user_mention["id"]  # TODO: Perhaps safe extract as well.
                mentioned_user_id_set.append(mentioned_user_id)

                id_to_name[mentioned_user_id] = user_mention["screen_name"]  # TODO: Perhaps safe extract as well.

            # We remove duplicates.
            mentioned_user_id_set = set(mentioned_user_id_set)

            # Get mentioned user ids.
            retweet_mentioned_user_id_set = list()
            if original_tweet_in_reply_to_user_id is not None:
                retweet_mentioned_user_id_set.append(original_tweet_in_reply_to_user_id)

                id_to_name[original_tweet_in_reply_to_user_id] = original_tweet_in_reply_to_screen_name
            for user_mention in original_tweet_entities_user_mentions:
                mentioned_user_id = user_mention["id"]  # TODO: Perhaps safe extract as well.
                retweet_mentioned_user_id_set.append(mentioned_user_id)

                id_to_name[mentioned_user_id] = user_mention["screen_name"]  # TODO: Perhaps safe extract as well.

            # We remove duplicates.
            retweet_mentioned_user_id_set = set(retweet_mentioned_user_id_set)

            mentioned_user_id_set.update(retweet_mentioned_user_id_set)

            # Update the mention graph one-by-one.
            for mentioned_user_id in mentioned_user_id_set:
                # Map users to distinct integer numbers.
                graph_size = len(id_to_node)
                mention_target_node = id_to_node.setdefault(mentioned_user_id, graph_size)

                append_user_id(mentioned_user_id)

                # Add values to the sparse matrix arrays.
                append_mention_graph_row(source_node)
                append_mention_graph_col(mention_target_node)

            # This is the first time we deal with this tweet.
            if original_tweet_id not in tweet_id_set:
                # Update sets, lists and dictionaries.
                add_tweet_id(original_tweet_id)
                id_to_name[original_tweet_user_id] = original_tweet_user_screen_name
                id_to_username[original_tweet_user_id] = original_tweet_user_name
                append_user_id(original_tweet_user_id)

                ########################################################################################################
                # Update mention matrix.
                ########################################################################################################
                # Update the mention graph one-by-one.
                for mentioned_user_id in retweet_mentioned_user_id_set:
                    # Map users to distinct integer numbers.
                    graph_size = len(id_to_node)
                    mention_target_node = id_to_node.setdefault(mentioned_user_id, graph_size)

                    append_user_id(mentioned_user_id)

                    # Add values to the sparse matrix arrays.
                    append_mention_graph_row(original_tweet_node)
                    append_mention_graph_col(mention_target_node)
            else:
                pass

    ####################################################################################################################
    # Final steps of preprocessing tweets.
    ####################################################################################################################
    # Discard any duplicates.
    user_id_set = set(user_id_set)
    number_of_users = len(user_id_set)
    # min_number_of_users = max(user_id_set) + 1

    # Form mention graph adjacency matrix.
    mention_graph_row = np.array(mention_graph_row, dtype=np.int64)
    mention_graph_col = np.array(mention_graph_col, dtype=np.int64)
    mention_graph_data = np.ones_like(mention_graph_row, dtype=np.float64)

    mention_graph = spsp.coo_matrix((mention_graph_data, (mention_graph_row, mention_graph_col)),
                                    shape=(number_of_users, number_of_users))
    mention_graph = spsp.coo_matrix(spsp.csr_matrix(mention_graph))

    # Form retweet graph adjacency matrix.
    retweet_graph_row = np.array(retweet_graph_row, dtype=np.int64)
    retweet_graph_col = np.array(retweet_graph_col, dtype=np.int64)
    retweet_graph_data = np.ones_like(retweet_graph_row, dtype=np.float64)

    retweet_graph = spsp.coo_matrix((retweet_graph_data, (retweet_graph_row, retweet_graph_col)),
                                    shape=(number_of_users, number_of_users))
    retweet_graph = spsp.coo_matrix(spsp.csr_matrix(retweet_graph))

    node_to_id = dict(zip(id_to_node.values(), id_to_node.keys()))

    return mention_graph, retweet_graph, tweet_id_set, user_id_set, node_to_id, id_to_name, id_to_username, id_to_listedcount