fix out-of-sequence ticks/bars

def _fix_history_sequence(self, df, table):
        """ fix out-of-sequence ticks/bars """

        # remove "Unnamed: x" columns
        cols = df.columns[df.columns.str.startswith('Unnamed:')].tolist()
        df.drop(cols, axis=1, inplace=True)

        # remove future dates
        df['datetime'] = pd.to_datetime(df['datetime'], utc=True)
        blacklist = df[df['datetime'] > pd.to_datetime('now', utc=True)]
        df = df.loc[set(df.index) - set(blacklist)]  # .tail()

        # loop through data, symbol by symbol
        dfs = []
        bad_ids = [blacklist['id'].values.tolist()]

        for symbol_id in list(df['symbol_id'].unique()):

            data = df[df['symbol_id'] == symbol_id].copy()

            # sort by id
            data.sort_values('id', axis=0, ascending=True, inplace=False)

            # convert index to column
            data.loc[:, "ix"] = data.index
            data.reset_index(inplace=True)

            # find out of sequence ticks/bars
            malformed = data.shift(1)[(data['id'] > data['id'].shift(1)) & (
                data['datetime'] < data['datetime'].shift(1))]

            # cleanup rows
            if malformed.empty:
                # if all rows are in sequence, just remove last row
                dfs.append(data)
            else:
                # remove out of sequence rows + last row from data
                index = [
                    x for x in data.index.values if x not in malformed['ix'].values]
                dfs.append(data.loc[index])

                # add to bad id list (to remove from db)
                bad_ids.append(list(malformed['id'].values))

        # combine all lists
        data = pd.concat(dfs, sort=True)

        # flatten bad ids
        bad_ids = sum(bad_ids, [])

        # remove bad ids from db
        if bad_ids:
            bad_ids = list(map(str, map(int, bad_ids)))
            self.dbcurr.execute("DELETE FROM greeks WHERE %s IN (%s)" % (
                table.lower()[:-1] + "_id", ",".join(bad_ids)))
            self.dbcurr.execute("DELETE FROM " + table.lower() +
                                " WHERE id IN (%s)" % (",".join(bad_ids)))
            try:
                self.dbconn.commit()
            except Exception as e:
                self.dbconn.rollback()

        # return
        return data.drop(['id', 'ix', 'index'], axis=1)