def add_covariate_to_timeline(
    long_form_df,
    cv,
    id_col,
    duration_col,
    event_col,
    start_col="start",
    stop_col="stop",
    add_enum=False,
    overwrite=True,
    cumulative_sum=False,
    cumulative_sum_prefix="cumsum_",
    delay=0,
):  # pylint: disable=too-many-arguments
    """
    This is a util function to help create a long form table tracking subjects' covariate changes over time. It is meant
    to be used iteratively as one adds more and more covariates to track over time. Before using this function, it is recommended
    to view the documentation at https://lifelines.readthedocs.io/en/latest/Survival%20Regression.html#dataset-creation-for-time-varying-regression.

    Parameters
    ----------
    long_form_df: DataFrame
        a DataFrame that has the initial or intermediate "long" form of time-varying observations. Must contain
        columns id_col, 'start', 'stop', and event_col. See function `to_long_format` to transform data into long form.
    cv: DataFrame
        a DataFrame that contains (possibly more than) one covariate to track over time. Must contain columns
        id_col and duration_col. duration_col represents time since the start of the subject's life.
    id_col: string
        the column in long_form_df and cv representing a unique identifier for subjects.
    duration_col: string
        the column in cv that represents the time-since-birth the observation occurred at.
    event_col: string
        the column in df that represents if the event-of-interest occurred
    add_enum: boolean, optional
         a Boolean flag to denote whether to add a column enumerating rows per subject. Useful to specify a specific
        observation, ex: df[df['enum'] == 1] will grab the first observations per subject.
    overwrite: boolean, optional
        if True, covariate values in long_form_df will be overwritten by covariate values in cv if the column exists in both
        cv and long_form_df and the timestamps are identical. If False, the default behavior will be to sum
        the values together.
    cumulative_sum: boolean, optional
        sum over time the new covariates. Makes sense if the covariates are new additions, and not state changes (ex:
        administering more drugs vs taking a temperature.)
    cumulative_sum_prefix: string, optional
        a prefix to add to calculated cumulative sum columns
    delay: int, optional
        add a delay to covariates (useful for checking for reverse causality in analysis)

    Returns
    -------
    long_form_df: DataFrame
        A DataFrame with updated rows to reflect the novel times slices (if any) being added from cv, and novel (or updated) columns
        of new covariates from cv

    See Also
    --------
    to_episodic_format
    to_long_format
    covariates_from_event_matrix
    """

    def remove_redundant_rows(cv):
        """
        Removes rows where no change occurs. Ex:

        cv = pd.DataFrame.from_records([
            {'id': 1, 't': 0, 'var3': 0, 'var4': 1},
            {'id': 1, 't': 1, 'var3': 0, 'var4': 1},  # redundant, as nothing changed during the interval
            {'id': 1, 't': 6, 'var3': 1, 'var4': 1},
        ])

        If cumulative_sum, then redundant rows are not redundant.
        """
        if cumulative_sum:
            return cv
        cols = cv.columns.difference([duration_col])
        cv = cv.loc[(cv[cols].shift() != cv[cols]).any(axis=1)]
        return cv

    def transform_cv_to_long_format(cv):
        return cv.rename(columns={duration_col: start_col})

    def expand(df, cvs):
        id_ = df.name
        try:
            cv = cvs.get_group(id_)
        except KeyError:
            return df

        final_state = bool(df[event_col].iloc[-1])
        final_stop_time = df[stop_col].iloc[-1]
        df = df.drop([id_col, event_col, stop_col], axis=1).set_index(start_col)
        cv = cv.drop([id_col], axis=1).set_index(start_col).loc[:final_stop_time]

        if cumulative_sum:
            cv = cv.cumsum()
            cv = cv.add_prefix(cumulative_sum_prefix)

        # How do I want to merge existing columns at the same time - could be
        # new observations (update) or new treatment applied (sum).
        # There may be more options in the future.
        if not overwrite:
            expanded_df = cv.combine(df, lambda s1, s2: s1 + s2, fill_value=0, overwrite=False)
        elif overwrite:
            expanded_df = cv.combine_first(df)

        n = expanded_df.shape[0]
        expanded_df = expanded_df.reset_index()
        expanded_df[stop_col] = expanded_df[start_col].shift(-1)
        expanded_df[id_col] = id_
        expanded_df[event_col] = False
        expanded_df.at[n - 1, event_col] = final_state
        expanded_df.at[n - 1, stop_col] = final_stop_time

        if add_enum:
            expanded_df["enum"] = np.arange(1, n + 1)

        if cumulative_sum:
            expanded_df[cv.columns] = expanded_df[cv.columns].ffill().fillna(0)

        return expanded_df.ffill()

    if delay < 0:
        raise ValueError("delay parameter must be equal to or greater than 0")

    if any(col not in long_form_df for col in (id_col, event_col, start_col, stop_col)):
        raise IndexError("Missing column in long_form_df")

    cv[duration_col] += delay
    cv = cv.dropna()
    cv = cv.sort_values([id_col, duration_col])
    cvs = cv.pipe(remove_redundant_rows).pipe(transform_cv_to_long_format).groupby(id_col)

    long_form_df = long_form_df.groupby(id_col, group_keys=False).apply(expand, cvs=cvs)
    return long_form_df.reset_index(drop=True)
def add_covariate_to_timeline ( long_form_df , cv , id_col , duration_col , event_col , start_col = "start" , stop_col = "stop" , add_enum = False , overwrite = True , cumulative_sum = False , cumulative_sum_prefix = "cumsum_" , delay = 0 , ) : def remove_redundant_rows ( cv ) : if cumulative_sum : return cv cols = cv . columns . difference ( [ duration_col ] ) cv = cv . loc [ ( cv [ cols ] . shift ( ) != cv [ cols ] ) . any ( axis = 1 ) ] return cv def transform_cv_to_long_format ( cv ) : return cv . rename ( columns = { duration_col : start_col } ) def expand ( df , cvs ) : id_ = df . name try : cv = cvs . get_group ( id_ ) except KeyError : return df final_state = bool ( df [ event_col ] . iloc [ - 1 ] ) final_stop_time = df [ stop_col ] . iloc [ - 1 ] df = df . drop ( [ id_col , event_col , stop_col ] , axis = 1 ) . set_index ( start_col ) cv = cv . drop ( [ id_col ] , axis = 1 ) . set_index ( start_col ) . loc [ : final_stop_time ] if cumulative_sum : cv = cv . cumsum ( ) cv = cv . add_prefix ( cumulative_sum_prefix ) if not overwrite : expanded_df = cv . combine ( df , lambda s1 , s2 : s1 + s2 , fill_value = 0 , overwrite = False ) elif overwrite : expanded_df = cv . combine_first ( df ) n = expanded_df . shape [ 0 ] expanded_df = expanded_df . reset_index ( ) expanded_df [ stop_col ] = expanded_df [ start_col ] . shift ( - 1 ) expanded_df [ id_col ] = id_ expanded_df [ event_col ] = False expanded_df . at [ n - 1 , event_col ] = final_state expanded_df . at [ n - 1 , stop_col ] = final_stop_time if add_enum : expanded_df [ "enum" ] = np . arange ( 1 , n + 1 ) if cumulative_sum : expanded_df [ cv . columns ] = expanded_df [ cv . columns ] . ffill ( ) . fillna ( 0 ) return expanded_df . ffill ( ) if delay < 0 : raise ValueError ( "delay parameter must be equal to or greater than 0" ) if any ( col not in long_form_df for col in ( id_col , event_col , start_col , stop_col ) ) : raise IndexError ( "Missing column in long_form_df" ) cv [ duration_col ] += delay cv = cv . dropna ( ) cv = cv . sort_values ( [ id_col , duration_col ] ) cvs = cv . pipe ( remove_redundant_rows ) . pipe ( transform_cv_to_long_format ) . groupby ( id_col ) long_form_df = long_form_df . groupby ( id_col , group_keys = False ) . apply ( expand , cvs = cvs ) return long_form_df . reset_index ( drop = True )
This is a util function to help create a long form table tracking subjects' covariate changes over time. It is meant
    to be used iteratively as one adds more and more covariates to track over time. Before using this function, it is recommended
    to view the documentation at https://lifelines.readthedocs.io/en/latest/Survival%20Regression.html#dataset-creation-for-time-varying-regression.

    Parameters
    ----------
    long_form_df: DataFrame
        a DataFrame that has the initial or intermediate "long" form of time-varying observations. Must contain
        columns id_col, 'start', 'stop', and event_col. See function `to_long_format` to transform data into long form.
    cv: DataFrame
        a DataFrame that contains (possibly more than) one covariate to track over time. Must contain columns
        id_col and duration_col. duration_col represents time since the start of the subject's life.
    id_col: string
        the column in long_form_df and cv representing a unique identifier for subjects.
    duration_col: string
        the column in cv that represents the time-since-birth the observation occurred at.
    event_col: string
        the column in df that represents if the event-of-interest occurred
    add_enum: boolean, optional
         a Boolean flag to denote whether to add a column enumerating rows per subject. Useful to specify a specific
        observation, ex: df[df['enum'] == 1] will grab the first observations per subject.
    overwrite: boolean, optional
        if True, covariate values in long_form_df will be overwritten by covariate values in cv if the column exists in both
        cv and long_form_df and the timestamps are identical. If False, the default behavior will be to sum
        the values together.
    cumulative_sum: boolean, optional
        sum over time the new covariates. Makes sense if the covariates are new additions, and not state changes (ex:
        administering more drugs vs taking a temperature.)
    cumulative_sum_prefix: string, optional
        a prefix to add to calculated cumulative sum columns
    delay: int, optional
        add a delay to covariates (useful for checking for reverse causality in analysis)

    Returns
    -------
    long_form_df: DataFrame
        A DataFrame with updated rows to reflect the novel times slices (if any) being added from cv, and novel (or updated) columns
        of new covariates from cv

    See Also
    --------
    to_episodic_format
    to_long_format
    covariates_from_event_matrix