INST_NAME_TO_MIDI_PROGRAM_DICT = {
  'violin': 41,
  'viola': 42,
  'cello': 43,
  'double bass': 44,
  'flute': 74,
  'oboe': 69,
  'clarinet': 72,
  'saxophone': 67,
  'bassoon': 71,
  'trumpet': 57,
  'horn': 61,
  'trombone': 58,
  'tuba': 59,
  'guitar': 27
} MIDI_PROGRAM_TO_INST_ID_DICT = {v: k for k, v in
                                INST_ID_TO_MIDI_PROGRAM_DICT.items()} MIDI_PROGRAM_TO_INST_NAME_DICT = {v: k for k, v in
                                  INST_NAME_TO_MIDI_PROGRAM_DICT.items()} def time_to_frame(time_s, fs=250):
def note_list_to_sequence(note_list, fs=250, pitch_offset=0, speed_rate=1,
                          remove_start_silence=False):
def mono_midi_to_note_sequence(midi_file, instrument_id, pitch_offset=0,
                               speed_rate=1):
def bach_midi_to_features(midi_file, quartet_set='woodwind_set',
                          quartet_set_dict=None, pitch_offset=0,
                          speed_rate=1):
def synthesize_bach(synthesis_generator, expression_generator, midi_file,
                    quartet_set='string_set', pitch_offset=0,
                    speed_rate=1, output_dir=r'./', gain_adjust_db_dict=None):
def batch_conditioning_df_to_audio(synthesis_generator, conditioning_df_all,
                                   instrument_id_all,
                                   display_progressbar=True):
def synthesize_mono_midi(synthesis_generator, expression_generator, midi_file,
                         instrument_id, output_dir,
                         pitch_offset=0,
                         speed_rate=1,
                         display_progressbar=True):
def fill_conditioning_df_with_mean(conditioning_df, note_expression_stat,
                                   instrument_abb):
def conditioning_df_to_f0_ld(conditioning_df, instrument_id, ld):
def conditioning_df_to_midi(conditioning_df, programs_id):
STRING_SET = {
  'Soprano': 'vn',
  'Alto': 'vn',
  'Tenor': 'va',
  'Bass': 'vc',
}
WOODWIND_SET = {
  'Soprano': 'fl',
  'Alto': 'ob',
  'Tenor': 'cl',
  'Bass': 'bn',
}
BRASSWIND_SET = {
  'Soprano': 'tpt',
  'Alto': 'hn',
  'Tenor': 'tbn',
  'Bass': 'tba',
}
MIX_SET = {
  'Soprano': 'cl',
  'Alto': 'tpt',
  'Tenor': 'sax',
  'Bass': 'tba',
}
QUARTET_SET_DICT = {
  'string_set': STRING_SET,
  'woodwind_set': WOODWIND_SET,
  'brasswind_set': BRASSWIND_SET,
  'mix_set': MIX_SET
} def save_wav(wav, path, sample_rate=16000):
  soundfile.write(path, wav, sample_rate) def get_hp(file):
  """Retrieve hyperparameters from log files."""
  with open(file, 'r') as f:
    log_all = f.readlines()
    f.close()
  hp = None
  for line in log_all:  # (yusongwu) hack here
    if '{\'add_synth_loss\':' in line:
      hp = literal_eval(line.strip()[32:])
      break
  if hp is None:
    raise RuntimeError('Failed to load the model.')
  return hp def ensure_same_length(data_all, axis=1):
  """Ensure the same length for all the data in the list."""
  max_length = max([d.shape[axis] for d in data_all])
  return [to_length(d, dst_length=max_length, axis=axis) for d in data_all] def pickle_dump(obj, path):
  with open(path, 'wb') as f:
    pickle.dump(obj, f)
    f.close() class hparams:
  """Hyperparameters for training Synthesis Generator."""
  # Learning parameters
  batch_size = 4
  clip_grad = 2.5
  lr = 3e-4
  seed = 1111

  # Training parameters
  training_steps = 100000
  log_interval = 100
  checkpoint_save_interval = 5000
  eval_interval = 5000
  mode = 'train'  # train, eval
  data_dir = None
  restore_path = None

  # Model Parameters
  nhid = 256
  sequence_length = 1000
  train_synth_coder_first = True
  midi_audio_loss = True
  add_synth_loss = False
  synth_params_loss = False
  midi_decoder_type = 'interpretable_conditioning'
  position_code = 'index_length'
  midi_decoder_decoder_net = 'rnn_synth_params'
  multi_instrument = True
  instrument = 'vn'
  synth_coder_training_steps = 300
  lambda_recon = 1.0
  use_gan = True
  lambda_G = 1
  sg_z = True
  lr_disc = 1e-4
  write_tfrecord_audio = False
  without_note_expression = False
  discriminator_dim = 256

  # Synthesis & DSP parameters
  nhramonic = 60
  nnoise = 65
  reverb = True
  reverb_length = 48000
  num_mels = 64
  n_fft = 1024
  sample_rate = 16000
  frame_size = 64
  hop_length = frame_size
  win_length = hop_length * 2
  frame_shift_ms = 1000 / sample_rate * frame_size
  fmin = 40 def get_synthesis_generator(hp):
  if hp.midi_decoder_decoder_net == 'rnn_f0_ld' or \
        hp.midi_decoder_type == 'midi_decoder' or \
        hp.midi_decoder_type == 'unconditioned':
    use_f0_ld = True
    encoder = F0LDEncoder()
    decoder = FCStackHarmonicDecoder(hp.nhramonic, hp.nnoise)
  else:
    cnn = Cnn8(pool_size=(1, 2))
    encoder = MelF0LDEncoder(cnn, hp.nhid, hp.sample_rate, hp.win_length,
                             hp.hop_length, hp.n_fft, hp.num_mels,
                             hp.fmin)
    decoder = FCHarmonicDecoder(hp.nhramonic, hp.nnoise)
    use_f0_ld = False
  synth_coder = SynthCoder(encoder, decoder)

  midi_decoder = get_midi_decoder(hp)

  if hp.reverb:
    num_reverb = NUM_INST if hp.multi_instrument else 1
    reverb_module = ReverbModules(num_reverb=num_reverb,
                                  reverb_length=hp.reverb_length)
  else:
    reverb_module = None

  model = MIDIExpressionAE(
    synth_coder=synth_coder,
    midi_decoder=midi_decoder,
    n_frames=hp.sequence_length,
    frame_size=hp.frame_size,
    sample_rate=hp.sample_rate,
    reverb_module=reverb_module,
    use_f0_ld=use_f0_ld
  )

  return model def get_fake_data_synthesis_generator(hp):
  """Get the fake data for building the synthesis generator."""
  fake_data = {
    'audio': tf.random.normal([1, hp.sequence_length * hp.frame_size]),
    'mel': tf.random.normal([1, hp.sequence_length, hp.num_mels]),
    'f0_hz': tf.random.normal([1, hp.sequence_length, 1]),
    'loudness_db': tf.random.normal([1, hp.sequence_length, 1]),
    'midi': tf.ones([1, hp.sequence_length], dtype=tf.int64),
    'instrument_id': tf.ones([1], dtype=tf.int64),
    'onsets': tf.zeros([1, hp.sequence_length], dtype=tf.int64),
    'offsets': tf.zeros([1, hp.sequence_length], dtype=tf.int64),
  }
  return fake_data, {'training': True} class ExpressionGenerator(TwoLayerCondAutoregRNN, tf.keras.Model, ABC):
  """Expression Generator that takes note sequence as input and predicts
  note expression controls."""

  # TODO：(yusongwu) merge teacher_force and autoregressive function,
  # things need to change:
  # def sample_out(self, out, cond, time, training=False):
  # curr_out = self.sample_out(curr_out, cond, i, training=training)
  # output = self.split_teacher_force_output(output, cond)
  def __init__(self, n_out=6, nhid=128, norm=True, dropout=0.5):
    super().__init__(
      nhid=nhid,
      n_out=n_out,
      input_dropout=True,
      input_dropout_p=0.5,
      dropout=dropout,
    )
    self.birnn = tfkl.Bidirectional(tfkl.GRU(
      units=nhid, return_sequences=True, dropout=dropout
    ), )
    self.dense_out = LangModelOutputLayer(nhid=nhid, n_out=self.n_out)
    self.norm = nn.Normalize('layer') if norm else None
    self.pitch_emb = tfkl.Embedding(128, 64)
    self.duration_emb = tfkl.Dense(64)
    self.instrument_emb = tfkl.Embedding(NUM_INST, 64)

  def autoregressive(self, cond, training=False):
    note_pitch = cond['note_pitch'][..., tf.newaxis]
    cond = self.encode_cond(cond, training=training)
    batch_size = cond.shape[0]
    length = cond.shape[1]
    prev_out = tf.tile([[0.0]], [batch_size, self.n_out])[:, tf.newaxis,
               :]  # go_frame
    prev_states = (None, None)
    overall_outputs = []

    for i in range(length):
      curr_cond = cond[:, i, :][:, tf.newaxis, :]
      prev_out = self.encode_out(prev_out)
      curr_out, curr_states = self._one_step(curr_cond, prev_out, prev_states,
                                             training=training)
      curr_out = self.sample_out(curr_out,
                                 note_pitch[:, i, :][:, tf.newaxis, :])
      overall_outputs.append(curr_out)
      prev_out, prev_states = curr_out['output'], curr_states

    outputs = {}
    for k in curr_out.keys():
      outputs[k] = tf.concat([x[k] for x in overall_outputs], 1)
    return outputs

  def teacher_force(self, cond, out, training=True):
    note_pitch = cond['note_pitch'][..., tf.newaxis]
    out_shifted = self.right_shift_encode_out(out)
    cond = self.encode_cond(cond, training=training)
    z_in = tf.concat([cond, out_shifted], -1)
    z_out, *states = self.rnn1(z_in, training=training)
    z_out, *states = self.rnn2(z_out, training=training)
    output = self.decode_out(z_out)
    output = self.sample_out(output, note_pitch)
    return output

  def encode_cond(self, cond, training=False):
    z_pitch = self.pitch_emb(cond['note_pitch'])
    z_duration = self.duration_emb(cond['note_length'])
    z_instrument = self.instrument_emb(
      tf.tile(cond['instrument_id'][:, tf.newaxis], [1, z_pitch.shape[1]]))
    cond = tf.concat([z_pitch, z_duration, z_instrument], -1)
    cond = self.birnn(cond, training=training)
    return cond

  def decode_out(self, z_out):
    if self.norm is not None:
      z_out = self.norm(z_out)
    output = self.dense_out(z_out)
    return output

  def sample_out(self, out, note_pitch):
    output = out.copy()
    sampled_output = out['raw_output']
    rest_note_mask = tf.cast(note_pitch != 0, tf.float32)
    sampled_output *= rest_note_mask
    output['output'] = sampled_output
    return output

  def call(self, cond, out=None, training=False):

    if training:
      outputs = self.teacher_force(cond, out, training=training)
    else:
      outputs = self.autoregressive(cond, training=training)
    return outputs def get_fake_data_expression_generator(target_dim):
  instrument_id = tf.ones([1], dtype=tf.int64)
  cond = {
    'note_pitch': tf.ones([1, 32], dtype=tf.int64),
    'note_length': tf.ones([1, 32, 1], dtype=tf.float32),
    'instrument_id': instrument_id
  }
  target = tf.ones([1, 32, target_dim], dtype=tf.float32)
  fake_data = {
    'cond': cond,
    'target': target
  }
  return fake_data
import os
import tensorflow as tf
import numpy as np
import pretty_midi
import argparse
import glob
from tqdm.autonotebook import tqdm
from midi_ddsp.data_handling.instrument_name_utils import \
  INST_NAME_TO_MIDI_PROGRAM_DICT, MIDI_PROGRAM_TO_INST_ID_DICT, \
  MIDI_PROGRAM_TO_INST_NAME_DICT
from midi_ddsp.utils.midi_synthesis_utils import note_list_to_sequence, \
  expression_generator_output_to_conditioning_df, batch_conditioning_df_to_audio
from midi_ddsp.utils.audio_io import save_wav
from midi_ddsp.utils.training_utils import get_hp
from midi_ddsp.utils.inference_utils import ensure_same_length
from midi_ddsp.utils.file_utils import pickle_dump
from midi_ddsp.hparams_synthesis_generator import hparams as hp
from midi_ddsp.modules.get_synthesis_generator import get_synthesis_generator, \
  get_fake_data_synthesis_generator
from midi_ddsp.modules.expression_generator import ExpressionGenerator, \
  get_fake_data_expression_generator
                            'Please run \'midi_ddsp_download_model_weights\' '
                            'to download model weights, '
                            'or specify path to model weights.')

  if synthesis_generator_path is None:
    synthesis_generator_path = os.path.join(
      package_dir,
      'midi_ddsp_model_weights_urmp_9_10',
      'synthesis_generator',
      '50000')
  if expression_generator_path is None:
    expression_generator_path = os.path.join(
      package_dir,
      'midi_ddsp_model_weights_urmp_9_10',
      'expression_generator',
      '5000')

  hp_dict = get_hp(
    os.path.join(os.path.dirname(synthesis_generator_path), 'train.log'))
  for k, v in hp_dict.items():
    setattr(hp, k, v)
  synthesis_generator = get_synthesis_generator(hp)
  synthesis_generator._build(get_fake_data_synthesis_generator(hp))
  synthesis_generator.load_weights(synthesis_generator_path).expect_partial()

  n_out = 6
  expression_generator = ExpressionGenerator(n_out=n_out, nhid=128)
  fake_data = get_fake_data_expression_generator(n_out)
  _ = expression_generator(fake_data['cond'], out=fake_data['target'],
                           training=True)
  expression_generator.load_weights(expression_generator_path).expect_partial()
  return synthesis_generator, expression_generator


def synthesize_midi(synthesis_generator, expression_generator, midi_file,
                    pitch_offset=0, speed_rate=1.0,
                    output_dir=None,
                    use_fluidsynth=False,
                    sf2_path='/usr/share/sounds/sf2/FluidR3_GM.sf2',
                    display_progressbar=True,
                    skip_existing_files=False,
                    save_synth_params=False):
  """
  Synthesize a midi file using MIDI-DDSP.
  Args:
      synthesis_generator: The instance of a Synthesis Generator.
      expression_generator: The instance of a expression generator.
      midi_file: The path to the MIDI file.
      pitch_offset: Pitch in semitone to transpose.
      speed_rate: The speed to synthesize the MIDI file.
      output_dir: The directory for saving outputs.
        If output_dir is None, outputs will not be written to disk.
      use_fluidsynth: Whether to use FluidSynth for synthesizing instruments
        that are not available in MIDI-DDSP.
      sf2_path: The path to a sf2 soundfont file used for FluidSynth.
      display_progressbar: Whether to display progress bar.
      skip_existing_files: Skip synthesizing MIDI files if already exist
        output folders.
      save_synth_params: also save synthesis parameters.

  Returns: A dict of output:
          'mix_audio': mix audio,
          'stem_audio': stem audios,
          'part_synth_by_model': list of part numbers that are synthesized
            by MIDI-DDSP,
          'midi_control_params': control parameters generated by MIDI-DDSP,
          'midi_synth_params': synth parameters generated by MIDI-DDSP,
          'conditioning_df': note expressions predicted by expression generator
            in the format of DataFrame,
  """
  # Check if there is existing files.
  filename = os.path.splitext(os.path.basename(midi_file))[0]
  if output_dir is not None:
    output_dir = os.path.join(output_dir, filename)
    if os.path.exists(output_dir) and skip_existing_files:
      print(f'{midi_file} has been synthesized, will skip this file.')
      return

  # Get all the midi program in URMP dataset excluding guitar.
  allowed_midi_program = list(INST_NAME_TO_MIDI_PROGRAM_DICT.values())[:-1]
  midi_data = pretty_midi.PrettyMIDI(midi_file)
  instrument_id_all = []
  conditioning_df_all = []
  part_synth_by_model = []
  midi_audio_all = {}
  midi_synth_params_all = {}
  midi_control_params_all = {}

  # For each part, predict expressions using MIDI-DDSP,
  # or synthesize using FluidSynth.
  for part_number, instrument in enumerate(midi_data.instruments):
    midi_program = instrument.program
    if midi_program in allowed_midi_program:
      note_sequence = note_list_to_sequence(instrument.notes, fs=FRAME_RATE,
                                            pitch_offset=pitch_offset,
                                            speed_rate=speed_rate)
      instrument_id = tf.constant([MIDI_PROGRAM_TO_INST_ID_DICT[midi_program]])
      instrument_id_all.append(instrument_id)
      note_sequence['instrument_id'] = instrument_id
      expression_generator_outputs = expression_generator(note_sequence,
                                                          out=None,
                                                          training=False)
      conditioning_df = expression_generator_output_to_conditioning_df(
        expression_generator_outputs['output'], note_sequence)
      conditioning_df_all.append(conditioning_df)
      part_synth_by_model.append(part_number)
    elif use_fluidsynth:
      instrument_name = MIDI_PROGRAM_TO_INST_NAME_DICT[midi_program]
      print(
        f'Part {part_number} in {midi_file} has {instrument_name} as '
        f'instrument which cannot be synthesized by model. '
        f'Using fluidsynth instead.')

      fluidsynth_wav_r3 = instrument.fluidsynth(SAMPLE_RATE, sf2_path=sf2_path)
      fluidsynth_wav_r3 *= 0.25  # * 0.25 for lower volume
      midi_audio_all[part_number] = fluidsynth_wav_r3

  # Synthesize audio in batch using Synthesis Generator.
  if len(conditioning_df_all) > 0:
    midi_audio, midi_control_params, midi_synth_params = \

      batch_conditioning_df_to_audio(