extract_melodies_features.py

import os
import pickle

import numpy as np
from tqdm import tqdm

# Environment settings
IS_COLAB = (os.name == 'posix')
LOAD_DATA = not (os.name == 'posix')

if not IS_COLAB:
    from prepare_data import create_validation_set


def get_midi_file_instrument_data(word_idx, time_per_word, midi_file):
    """
    Extract data about the midi file in the given time period. We will extract number of beat changes, instruments used
    and velocity.
    :param word_idx: index of word in the song
    :param time_per_word: Average time per word in song
    :param midi_file: The midi file
    :return: An array where each cell contains some data about the pitch, velocity etc.
    """
    #  Features we want to extract:
    start_time = word_idx * time_per_word
    end_time = start_time + time_per_word
    avg_velocity, avg_pitch, num_of_instruments, num_of_notes, beat_changes, has_drums = 0, 0, 0, 0, 0, 0

    for beat in midi_file.get_beats():
        if start_time <= beat <= end_time:
            beat_changes += 1  # Count beats that are in the desired time frame
        elif beat > end_time:
            break  # We passed the final possible time
    for instrument in midi_file.instruments:
        in_range = False  # Will become true if the instrument contributed at least 1 note for this sequence.
        for note in instrument.notes:
            if start_time <= note.start:
                if note.end <= end_time:  # In required range
                    has_drums = 1 if instrument.is_drum else has_drums
                    in_range = True
                    num_of_notes += 1
                    avg_pitch += note.pitch
                    avg_velocity += note.velocity
                else:  # We passed the last relevant note
                    break
        if in_range:
            num_of_instruments += 1
    if num_of_notes > 0:  # If there was at least 1 note
        avg_velocity /= num_of_notes
        avg_pitch /= num_of_notes
    final_features = np.array([avg_velocity, avg_pitch, num_of_instruments, beat_changes, has_drums])
    return final_features


def extract_melody_features_1(melodies_list, sequence_length, encoded_song_lyrics):
    """
    First function for extracting features about the midi files. Using the instrument objects in each midi file we can
    see when each instrument was used and with what velocity. We can then calculate the average pitch and velocity for
    each word in the song.
    
    :param melodies_list: A list of midi files. Contains the training / validation / test set typically.
    :param sequence_length: Number of words per sequence.
    :param encoded_song_lyrics: A list where each cell represents a song. The cells contain a list of ints, where each
    cell corresponds to a word in the songs lyrics and the value is the index of the word in our word2vec vocabulary.
    :return: A 3d numpy array where the first axis is the number of sequences in the data, the 2nd is the sequence
    length and the third is the number of notes for that particular word in that sequence.
    """

    final_features = []
    print('Extracting melody features v1..')

    for idx, midi_file in tqdm(enumerate(melodies_list)):
        num_of_words_in_song = len(encoded_song_lyrics[idx])
        midi_file.remove_invalid_notes()
        time_per_word = midi_file.get_end_time() / num_of_words_in_song  # Average time per word in the lyrics
        number_of_sequences = num_of_words_in_song - sequence_length
        features_during_lyric = []
        for word_idx in range(num_of_words_in_song):  # Iterate over every word and get the features for it
            instrument_data = get_midi_file_instrument_data(word_idx, time_per_word, midi_file)
            features_during_lyric.append(instrument_data)

        for sequence_num in range(number_of_sequences):
            seq = features_during_lyric[sequence_num:sequence_num + sequence_length]  # Create a sequence from the notes
            final_features.append(seq)

    final_features = np.array(final_features)
    return final_features


def extract_melody_features_2(melodies_list, sequence_length, encoded_song_lyrics):
    """
    Using all midi files and lyrics, extract features for all sequences. This is the second method we'll try. Basically,
    we will take the piano roll matrix for each song. This is a matrix that displays which notes were played for every
    user defined time period and some number representing the velocity. In our case, we'll slice the song every 1/50
    seconds (20 miliseconds) and look at what notes were played during this time. This is in addition to the features
    used in v1.
    :param melodies_list: A list of midi files. Contains the training / validation / test set typically.
    :param total_dataset_size: Total length of the sequence array,
    :param sequence_length: Number of words per sequence.
    :param encoded_song_lyrics: A list where each cell represents a song. The cells contain a list of ints, where each cell
    corresponds to a word in the songs lyrics and the value is the index of the word in our word2vec vocabulary.
    :return: A 3d numpy array where the first axis is the number of sequences in the data, the 2nd is the sequence
    length and the third is the number of notes for that particular word in that sequence.
    """

    final_features = []
    print('Extracting melody features v2..')
    frequency_sample = 50
    for midi_idx, midi_file in tqdm(enumerate(melodies_list)):
        num_of_words_in_song = len(encoded_song_lyrics[midi_idx])
        midi_file.remove_invalid_notes()
        time_per_word = midi_file.get_end_time() / num_of_words_in_song  # Average time per word in the lyrics
        number_of_sequences = num_of_words_in_song - sequence_length
        piano_roll = midi_file.get_piano_roll(fs=frequency_sample)
        num_of_notes_per_word = int(piano_roll.shape[1] / num_of_words_in_song)  # Num of piano roll columns per word
        features_during_lyric = []
        for word_idx in range(num_of_words_in_song):   # Iterate over every word and get the features for it
            notes_features = extract_piano_roll_features(num_of_notes_per_word, piano_roll, word_idx)
            instrument_data = get_midi_file_instrument_data(word_idx, time_per_word, midi_file)
            features = np.append(notes_features, instrument_data, axis=0)  # Concatenate them
            features_during_lyric.append(features)

        for sequence_num in range(number_of_sequences):
            # Create the features per sequence
            sequence_features = features_during_lyric[sequence_num:sequence_num + sequence_length]
            final_features.append(sequence_features)

    final_features = np.array(final_features)
    return final_features


def extract_piano_roll_features(num_of_notes_per_word, piano_roll, word_idx):
    start_idx = word_idx * num_of_notes_per_word
    end_idx = start_idx + num_of_notes_per_word
    piano_roll_for_lyric = piano_roll[:, start_idx:end_idx].transpose()
    piano_roll_slice_sum = np.sum(piano_roll_for_lyric, axis=0)  # Sum each column into a single cell
    return piano_roll_slice_sum


def get_melody_data_sets(train_num, val_size, melodies_list, sequence_length, encoded_lyrics_matrix, seed,
                         pkl_file_path, feature_method):
    """
    Creates numpy arrays containing features of the melody for the training, validation and test sets.
    :param feature_method: Method of feature extraction to use. Either '1' or '2'.
    :param seed: Seed for splitting to train and test.
    :param pkl_file_path: the file path to the pickle file. Used for saving or loading.
    :param train_num: Number of words in the whole training set sequence (train + validation)
    :param val_size: Percentage of sequences used for validation set
    :param melodies_list: All of the training + validation set midi files
    :param sequence_length: Number of words in a sequence
    :param encoded_lyrics_matrix: A list where each cell represents a song. The cells contain a list of ints, where each cell
    corresponds to a word in the songs lyrics and the value is the index of the word in our word2vec vocabulary.
    :return: numpy arrays containing features of the melody for the training, validation and test sets.
    """
    file_type = pkl_file_path.split('.')[-1]
    # Save/load the file with the appropriate name according to the settings used:
    pkl_file_path = f'{pkl_file_path.rstrip("." + file_type)}_{str(feature_method)}_sl_{sequence_length}.{file_type}'
    if os.path.exists(pkl_file_path):  # If file exists, use it instead of building it again
        with open(pkl_file_path, 'rb') as f:
            melody_train, melody_val, melody_test = pickle.load(f)
        return melody_train, melody_val, melody_test

    if feature_method == 'naive':  # Use appropriate melody feature method
        melody_features = extract_melody_features_1(melodies_list, sequence_length, encoded_lyrics_matrix)
    else:
        melody_features = extract_melody_features_2(melodies_list, sequence_length, encoded_lyrics_matrix)

    melody_train = melody_features[:train_num]
    melody_test = melody_features[train_num:]
    melody_train, melody_val = create_validation_set(melody_train, val_size, seed)

    with open(pkl_file_path, 'wb') as f:
        pickle.dump([melody_train, melody_val, melody_test], f)
        print('Dumped midi files')

    return melody_train, melody_val, melody_test


print("Loaded Successfully")