Upload 6 files

checkpoints/log_wsj0-2mix_speech_SpEx-plus_2spk/config.yaml

@@ -0,0 +1,44 @@
+## Config file
+
+# Log 
+seed: 777
+use_cuda: 1           # 1 for True, 0 for False
+
+# dataset
+speaker_no: 2
+mix_lst_path: ./data/wsj0_2mix/
+audio_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/
+reference_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/
+audio_sr: 8000
+ref_sr: 8000
+
+# dataloader
+num_workers: 4
+batch_size: 4         # 2-GPU training with a total effective batch size of 8
+accu_grad: 0
+effec_batch_size: 4   # per GPU, only used if accu_grad is set to 1, must be multiple times of batch size
+max_length: 4         # truncate the utterances in dataloader, in seconds 
+
+# network settings
+init_from: None       # 'None' or a log name 'log_2024-07-22(18:12:13)'
+causal: 0             # 1 for True, 0 for False
+network_reference:
+  cue: speech            # lip or speech or gesture or EEG
+network_audio:
+  backbone: SpEx-plus
+  L: 20
+  N: 256
+  X: 8
+  R: 4
+  B: 256
+  H: 512
+  P: 3
+  norm: gLN
+  non_linear: relu
+  speakers: 101       # 101 speakers in wsj0-2mix training set
+
+# optimizer
+loss_type: SpEx-plus     # spex loss in paper
+init_learning_rate: 0.001
+max_epoch: 200
+clip_grad_norm: 5

checkpoints/log_wsj0-2mix_speech_SpEx-plus_2spk/last_best_checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ffcf87a45f46ece3fa43db5b4d7f9779a73392933fed0b563f0ead9bd9b492f
+size 134255410

checkpoints/log_wsj0-2mix_speech_SpEx-plus_2spk/last_checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2399b66b4eb8eab7c4c31482a20c24cfb92f0c49682400b97c5d5eb8d6b8b69f
+size 134246515

checkpoints/log_wsj0-2mix_speech_SpEx-plus_2spk/log_2024-10-02(16:13:54).txt

@@ -0,0 +1,803 @@
+## Config file
+
+# Log 
+seed: 777
+use_cuda: 1           # 1 for True, 0 for False
+
+# dataset
+speaker_no: 2
+mix_lst_path: ./data/wsj0_2mix/
+audio_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/
+reference_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/
+audio_sr: 8000
+ref_sr: 8000
+
+# dataloader
+num_workers: 4
+batch_size: 4         # 2-GPU training with a total effective batch size of 8
+accu_grad: 0
+effec_batch_size: 4   # per GPU, only used if accu_grad is set to 1, must be multiple times of batch size
+max_length: 4         # truncate the utterances in dataloader, in seconds 
+
+# network settings
+init_from: None       # 'None' or a log name 'log_2024-07-22(18:12:13)'
+causal: 0             # 1 for True, 0 for False
+network_reference:
+  cue: speech            # lip or speech or gesture or EEG
+network_audio:
+  backbone: SpEx-plus
+  L: 20
+  N: 256
+  X: 8
+  R: 4
+  B: 256
+  H: 512
+  P: 3
+  norm: gLN
+  non_linear: relu
+  speakers: 101       # 101 speakers in wsj0-2mix training set
+
+# optimizer
+loss_type: SpEx-plus     # spex loss in paper
+init_learning_rate: 0.001
+max_epoch: 200
+clip_grad_norm: 5
+W1002 16:13:58.402247 140563653224256 torch/distributed/run.py:779] 
+W1002 16:13:58.402247 140563653224256 torch/distributed/run.py:779] *****************************************
+W1002 16:13:58.402247 140563653224256 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W1002 16:13:58.402247 140563653224256 torch/distributed/run.py:779] *****************************************
+[W1002 16:14:02.131922970 Utils.hpp:135] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
+[W1002 16:14:02.132797365 Utils.hpp:135] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
+started on checkpoints/log_2024-10-02(16:13:54)
+
+namespace(accu_grad=0, audio_direc='/mnt/nas_sg/wulanchabu/zexu.pan/datasets/', audio_sr=8000, batch_size=4, causal=0, checkpoint_dir='checkpoints/log_2024-10-02(16:13:54)', clip_grad_norm=5.0, config=[<yamlargparse.Path object at 0x7feab35cbd00>], device=device(type='cuda'), distributed=True, effec_batch_size=4, init_from='None', init_learning_rate=0.001, local_rank=0, loss_type='SpEx-plus', lr_warmup=0, max_epoch=200, max_length=4, mix_lst_path='./data/wsj0_2mix/', network_audio=namespace(B=256, H=512, L=20, N=256, P=3, R=4, X=8, backbone='SpEx-plus', non_linear='relu', norm='gLN', speakers=101), network_reference=namespace(cue='speech'), num_workers=4, ref_sr=8000, reference_direc='/mnt/nas_sg/wulanchabu/zexu.pan/datasets/', seed=777, speaker_no=2, train_from_last_checkpoint=0, use_cuda=1, world_size=2)
+network_wrapper(
+  (sep_network): SpEx_plus(
+    (encoder_1d_short): Conv1D(1, 256, kernel_size=(20,), stride=(10,))
+    (encoder_1d_middle): Conv1D(1, 256, kernel_size=(80,), stride=(10,))
+    (encoder_1d_long): Conv1D(1, 256, kernel_size=(160,), stride=(10,))
+    (ln): ChannelWiseLayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (proj): Conv1D(768, 256, kernel_size=(1,), stride=(1,))
+    (conv_block_1): Conv1DBlock_v2(
+      (conv1x1): Conv1D(512, 512, kernel_size=(1,), stride=(1,))
+      (prelu1): PReLU(num_parameters=1)
+      (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+      (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+      (prelu2): PReLU(num_parameters=1)
+      (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+      (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+    )
+    (conv_block_1_other): Sequential(
+      (0): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(2,), dilation=(2,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (1): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(4,), dilation=(4,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (2): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(8,), dilation=(8,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (3): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(16,), dilation=(16,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (4): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(32,), dilation=(32,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (5): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(64,), dilation=(64,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (6): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(128,), dilation=(128,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+    )
+    (conv_block_2): Conv1DBlock_v2(
+      (conv1x1): Conv1D(512, 512, kernel_size=(1,), stride=(1,))
+      (prelu1): PReLU(num_parameters=1)
+      (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+      (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+      (prelu2): PReLU(num_parameters=1)
+      (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+      (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+    )
+    (conv_block_2_other): Sequential(
+      (0): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(2,), dilation=(2,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (1): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(4,), dilation=(4,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (2): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(8,), dilation=(8,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (3): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(16,), dilation=(16,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (4): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(32,), dilation=(32,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (5): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(64,), dilation=(64,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (6): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(128,), dilation=(128,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+    )
+    (conv_block_3): Conv1DBlock_v2(
+      (conv1x1): Conv1D(512, 512, kernel_size=(1,), stride=(1,))
+      (prelu1): PReLU(num_parameters=1)
+      (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+      (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+      (prelu2): PReLU(num_parameters=1)
+      (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+      (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+    )
+    (conv_block_3_other): Sequential(
+      (0): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(2,), dilation=(2,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (1): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(4,), dilation=(4,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (2): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(8,), dilation=(8,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (3): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(16,), dilation=(16,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (4): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(32,), dilation=(32,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (5): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(64,), dilation=(64,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (6): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(128,), dilation=(128,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+    )
+    (conv_block_4): Conv1DBlock_v2(
+      (conv1x1): Conv1D(512, 512, kernel_size=(1,), stride=(1,))
+      (prelu1): PReLU(num_parameters=1)
+      (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+      (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+      (prelu2): PReLU(num_parameters=1)
+      (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+      (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+    )
+    (conv_block_4_other): Sequential(
+      (0): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(2,), dilation=(2,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (1): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(4,), dilation=(4,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (2): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(8,), dilation=(8,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (3): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(16,), dilation=(16,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (4): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(32,), dilation=(32,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (5): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(64,), dilation=(64,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+      (6): Conv1DBlock(
+        (conv1x1): Conv1D(256, 512, kernel_size=(1,), stride=(1,))
+        (prelu1): PReLU(num_parameters=1)
+        (lnorm1): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (dconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(128,), dilation=(128,), groups=512)
+        (prelu2): PReLU(num_parameters=1)
+        (lnorm2): GlobalChannelLayerNorm(512, eps=1e-05, elementwise_affine=True)
+        (sconv): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+      )
+    )
+    (mask1): Conv1D(256, 256, kernel_size=(1,), stride=(1,))
+    (mask2): Conv1D(256, 256, kernel_size=(1,), stride=(1,))
+    (mask3): Conv1D(256, 256, kernel_size=(1,), stride=(1,))
+    (decoder_1d_1): ConvTrans1D(256, 1, kernel_size=(20,), stride=(10,))
+    (decoder_1d_2): ConvTrans1D(256, 1, kernel_size=(80,), stride=(10,))
+    (decoder_1d_3): ConvTrans1D(256, 1, kernel_size=(160,), stride=(10,))
+    (aux_enc3): Sequential(
+      (0): ChannelWiseLayerNorm((768,), eps=1e-05, elementwise_affine=True)
+      (1): Conv1D(768, 256, kernel_size=(1,), stride=(1,))
+      (2): ResBlock(
+        (conv1): Conv1d(256, 256, kernel_size=(1,), stride=(1,), bias=False)
+        (conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,), bias=False)
+        (batch_norm1): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (batch_norm2): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (prelu1): PReLU(num_parameters=1)
+        (prelu2): PReLU(num_parameters=1)
+        (mp): MaxPool1d(kernel_size=3, stride=3, padding=0, dilation=1, ceil_mode=False)
+      )
+      (3): ResBlock(
+        (conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (conv2): Conv1d(512, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (batch_norm1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (batch_norm2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (prelu1): PReLU(num_parameters=1)
+        (prelu2): PReLU(num_parameters=1)
+        (mp): MaxPool1d(kernel_size=3, stride=3, padding=0, dilation=1, ceil_mode=False)
+        (conv_downsample): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (4): ResBlock(
+        (conv1): Conv1d(512, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (conv2): Conv1d(512, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (batch_norm1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (batch_norm2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (prelu1): PReLU(num_parameters=1)
+        (prelu2): PReLU(num_parameters=1)
+        (mp): MaxPool1d(kernel_size=3, stride=3, padding=0, dilation=1, ceil_mode=False)
+      )
+      (5): Conv1D(512, 256, kernel_size=(1,), stride=(1,))
+    )
+    (pred_linear): Linear(in_features=256, out_features=101, bias=True)
+  )
+)
+
+Total number of parameters: 11138734 
+
+
+Total number of trainable parameters: 11138734 
+
+Initialised Softmax Loss
+Initialised Softmax Loss
+dsw-106518-965b74ddc-cdclg:3549648:3549648 [0] NCCL INFO Bootstrap : Using net0:10.32.15.154<0>
+dsw-106518-965b74ddc-cdclg:3549648:3549648 [0] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
+dsw-106518-965b74ddc-cdclg:3549648:3549648 [0] NCCL INFO cudaDriverVersion 11040
+NCCL version 2.20.5+cuda11.8
+dsw-106518-965b74ddc-cdclg:3549649:3549649 [1] NCCL INFO cudaDriverVersion 11040
+dsw-106518-965b74ddc-cdclg:3549649:3549649 [1] NCCL INFO Bootstrap : Using net0:10.32.15.154<0>
+dsw-106518-965b74ddc-cdclg:3549649:3549649 [1] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO Failed to open libibverbs.so[.1]
+dsw-106518-965b74ddc-cdclg:3549649:3549740 [1] NCCL INFO Failed to open libibverbs.so[.1]
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO NET/Socket : Using [0]net0:10.32.15.154<0> [1]eth0:22.5.146.138<0>
+dsw-106518-965b74ddc-cdclg:3549649:3549740 [1] NCCL INFO NET/Socket : Using [0]net0:10.32.15.154<0> [1]eth0:22.5.146.138<0>
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO Using non-device net plugin version 0
+dsw-106518-965b74ddc-cdclg:3549649:3549740 [1] NCCL INFO Using non-device net plugin version 0
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO Using network Socket
+dsw-106518-965b74ddc-cdclg:3549649:3549740 [1] NCCL INFO Using network Socket
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO comm 0x78839c0 rank 0 nranks 2 cudaDev 0 nvmlDev 0 busId 10 commId 0x646868210b4fe2c6 - Init START
+dsw-106518-965b74ddc-cdclg:3549649:3549740 [1] NCCL INFO comm 0x8e72ac0 rank 1 nranks 2 cudaDev 1 nvmlDev 1 busId 20 commId 0x646868210b4fe2c6 - Init START
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO Setting affinity for GPU 0 to ffffff
+dsw-106518-965b74ddc-cdclg:3549649:3549740 [1] NCCL INFO Setting affinity for GPU 1 to ffffff
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO comm 0x78839c0 rank 0 nRanks 2 nNodes 1 localRanks 2 localRank 0 MNNVL 0
+dsw-106518-965b74ddc-cdclg:3549649:3549740 [1] NCCL INFO comm 0x8e72ac0 rank 1 nRanks 2 nNodes 1 localRanks 2 localRank 1 MNNVL 0
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO NCCL_MAX_NCHANNELS set by environment to 2.
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 2.
+dsw-106518-965b74ddc-cdclg:3549649:3549740 [1] NCCL INFO NCCL_MAX_NCHANNELS set by environment to 2.
+dsw-106518-965b74ddc-cdclg:3549649:3549740 [1] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 2.
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO Channel 00/02 :    0   1
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO Channel 01/02 :    0   1
+dsw-106518-965b74ddc-cdclg:3549649:3549740 [1] NCCL INFO Trees [0] -1/-1/-1->1->0 [1] 0/-1/-1->1->-1
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] -1/-1/-1->0->1
+dsw-106518-965b74ddc-cdclg:3549649:3549740 [1] NCCL INFO P2P Chunksize set to 524288
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO P2P Chunksize set to 524288
+dsw-106518-965b74ddc-cdclg:3549649:3549740 [1] NCCL INFO Channel 00/0 : 1[1] -> 0[0] via P2P/IPC/read
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/IPC/read
+dsw-106518-965b74ddc-cdclg:3549649:3549740 [1] NCCL INFO Channel 01/0 : 1[1] -> 0[0] via P2P/IPC/read
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/IPC/read
+dsw-106518-965b74ddc-cdclg:3549649:3549740 [1] NCCL INFO Connected all rings
+dsw-106518-965b74ddc-cdclg:3549649:3549740 [1] NCCL INFO Connected all trees
+dsw-106518-965b74ddc-cdclg:3549649:3549740 [1] NCCL INFO threadThresholds 8/8/64 | 16/8/64 | 512 | 512
+dsw-106518-965b74ddc-cdclg:3549649:3549740 [1] NCCL INFO 2 coll channels, 0 collnet channels, 0 nvls channels, 2 p2p channels, 2 p2p channels per peer
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO Connected all rings
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO Connected all trees
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO threadThresholds 8/8/64 | 16/8/64 | 512 | 512
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO 2 coll channels, 0 collnet channels, 0 nvls channels, 2 p2p channels, 2 p2p channels per peer
+dsw-106518-965b74ddc-cdclg:3549649:3549740 [1] NCCL INFO NCCL_LAUNCH_MODE set by environment to PARALLEL
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO NCCL_LAUNCH_MODE set by environment to PARALLEL
+dsw-106518-965b74ddc-cdclg:3549648:3549739 [0] NCCL INFO comm 0x78839c0 rank 0 nranks 2 cudaDev 0 nvmlDev 0 busId 10 commId 0x646868210b4fe2c6 - Init COMPLETE
+dsw-106518-965b74ddc-cdclg:3549649:3549740 [1] NCCL INFO comm 0x8e72ac0 rank 1 nranks 2 cudaDev 1 nvmlDev 1 busId 20 commId 0x646868210b4fe2c6 - Init COMPLETE
+[rank0]:[W1002 16:14:09.364378929 Utils.hpp:110] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
+Start new training from scratch
+[rank1]:[W1002 16:14:09.364850197 Utils.hpp:110] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
+[rank0]:[W1002 16:14:16.105722831 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+[rank1]:[W1002 16:14:16.124629939 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+Train Summary | End of Epoch 1 | Time 1938.77s | Train Loss -5.137
+Valid Summary | End of Epoch 1 | Time 127.26s | Valid Loss -8.814
+Test Summary | End of Epoch 1 | Time 77.56s | Test Loss -9.161
+Fund new best model, dict saved
+Train Summary | End of Epoch 2 | Time 1938.88s | Train Loss -10.133
+Valid Summary | End of Epoch 2 | Time 126.07s | Valid Loss -9.820
+Test Summary | End of Epoch 2 | Time 75.90s | Test Loss -10.366
+Fund new best model, dict saved
+Train Summary | End of Epoch 3 | Time 1937.57s | Train Loss -11.814
+Valid Summary | End of Epoch 3 | Time 125.85s | Valid Loss -11.866
+Test Summary | End of Epoch 3 | Time 76.05s | Test Loss -11.050
+Fund new best model, dict saved
+Train Summary | End of Epoch 4 | Time 1931.60s | Train Loss -12.824
+Valid Summary | End of Epoch 4 | Time 126.25s | Valid Loss -12.386
+Test Summary | End of Epoch 4 | Time 75.53s | Test Loss -11.700
+Fund new best model, dict saved
+Train Summary | End of Epoch 5 | Time 1934.21s | Train Loss -13.589
+Valid Summary | End of Epoch 5 | Time 126.16s | Valid Loss -13.496
+Test Summary | End of Epoch 5 | Time 75.61s | Test Loss -12.582
+Fund new best model, dict saved
+Train Summary | End of Epoch 6 | Time 1935.56s | Train Loss -14.183
+Valid Summary | End of Epoch 6 | Time 126.10s | Valid Loss -13.982
+Test Summary | End of Epoch 6 | Time 75.56s | Test Loss -13.381
+Fund new best model, dict saved
+Train Summary | End of Epoch 7 | Time 1933.10s | Train Loss -14.677
+Valid Summary | End of Epoch 7 | Time 126.04s | Valid Loss -14.077
+Test Summary | End of Epoch 7 | Time 75.73s | Test Loss -13.426
+Fund new best model, dict saved
+Train Summary | End of Epoch 8 | Time 1933.57s | Train Loss -15.064
+Valid Summary | End of Epoch 8 | Time 126.47s | Valid Loss -14.802
+Test Summary | End of Epoch 8 | Time 76.01s | Test Loss -13.831
+Fund new best model, dict saved
+Train Summary | End of Epoch 9 | Time 1936.63s | Train Loss -15.402
+Valid Summary | End of Epoch 9 | Time 125.85s | Valid Loss -15.042
+Test Summary | End of Epoch 9 | Time 75.88s | Test Loss -14.217
+Fund new best model, dict saved
+Train Summary | End of Epoch 10 | Time 1933.70s | Train Loss -15.696
+Valid Summary | End of Epoch 10 | Time 125.93s | Valid Loss -15.002
+Test Summary | End of Epoch 10 | Time 75.76s | Test Loss -13.852
+Train Summary | End of Epoch 11 | Time 1934.22s | Train Loss -15.956
+Valid Summary | End of Epoch 11 | Time 125.82s | Valid Loss -15.403
+Test Summary | End of Epoch 11 | Time 75.53s | Test Loss -14.515
+Fund new best model, dict saved
+Train Summary | End of Epoch 12 | Time 1934.42s | Train Loss -16.151
+Valid Summary | End of Epoch 12 | Time 125.82s | Valid Loss -15.502
+Test Summary | End of Epoch 12 | Time 75.47s | Test Loss -14.775
+Fund new best model, dict saved
+Train Summary | End of Epoch 13 | Time 1935.17s | Train Loss -16.377
+Valid Summary | End of Epoch 13 | Time 125.95s | Valid Loss -15.679
+Test Summary | End of Epoch 13 | Time 75.78s | Test Loss -14.593
+Fund new best model, dict saved
+Train Summary | End of Epoch 14 | Time 1933.74s | Train Loss -16.557
+Valid Summary | End of Epoch 14 | Time 126.33s | Valid Loss -15.069
+Test Summary | End of Epoch 14 | Time 76.02s | Test Loss -14.342
+Train Summary | End of Epoch 15 | Time 1932.98s | Train Loss -16.746
+Valid Summary | End of Epoch 15 | Time 125.66s | Valid Loss -16.075
+Test Summary | End of Epoch 15 | Time 75.63s | Test Loss -14.788
+Fund new best model, dict saved
+Train Summary | End of Epoch 16 | Time 1935.18s | Train Loss -16.889
+Valid Summary | End of Epoch 16 | Time 125.97s | Valid Loss -16.026
+Test Summary | End of Epoch 16 | Time 75.36s | Test Loss -14.844
+Train Summary | End of Epoch 17 | Time 1937.49s | Train Loss -17.054
+Valid Summary | End of Epoch 17 | Time 126.31s | Valid Loss -16.395
+Test Summary | End of Epoch 17 | Time 75.79s | Test Loss -15.114
+Fund new best model, dict saved
+Train Summary | End of Epoch 18 | Time 1934.17s | Train Loss -17.182
+Valid Summary | End of Epoch 18 | Time 125.78s | Valid Loss -16.482
+Test Summary | End of Epoch 18 | Time 75.59s | Test Loss -15.295
+Fund new best model, dict saved
+Train Summary | End of Epoch 19 | Time 1932.89s | Train Loss -17.311
+Valid Summary | End of Epoch 19 | Time 125.77s | Valid Loss -16.488
+Test Summary | End of Epoch 19 | Time 75.90s | Test Loss -15.158
+Fund new best model, dict saved
+Train Summary | End of Epoch 20 | Time 1935.75s | Train Loss -17.433
+Valid Summary | End of Epoch 20 | Time 125.57s | Valid Loss -16.730
+Test Summary | End of Epoch 20 | Time 75.65s | Test Loss -15.259
+Fund new best model, dict saved
+Train Summary | End of Epoch 21 | Time 1933.86s | Train Loss -17.535
+Valid Summary | End of Epoch 21 | Time 125.81s | Valid Loss -16.474
+Test Summary | End of Epoch 21 | Time 76.08s | Test Loss -15.619
+Train Summary | End of Epoch 22 | Time 1358.25s | Train Loss -17.638
+Valid Summary | End of Epoch 22 | Time 68.66s | Valid Loss -16.645
+Test Summary | End of Epoch 22 | Time 38.26s | Test Loss -15.025
+Train Summary | End of Epoch 23 | Time 815.45s | Train Loss -17.745
+Valid Summary | End of Epoch 23 | Time 62.66s | Valid Loss -16.885
+Test Summary | End of Epoch 23 | Time 38.78s | Test Loss -15.387
+Fund new best model, dict saved
+Train Summary | End of Epoch 24 | Time 812.33s | Train Loss -17.824
+Valid Summary | End of Epoch 24 | Time 62.43s | Valid Loss -16.871
+Test Summary | End of Epoch 24 | Time 38.44s | Test Loss -15.098
+Train Summary | End of Epoch 25 | Time 812.85s | Train Loss -17.911
+Valid Summary | End of Epoch 25 | Time 62.76s | Valid Loss -16.963
+Test Summary | End of Epoch 25 | Time 38.86s | Test Loss -15.535
+Fund new best model, dict saved
+Train Summary | End of Epoch 26 | Time 812.48s | Train Loss -18.000
+Valid Summary | End of Epoch 26 | Time 63.16s | Valid Loss -17.169
+Test Summary | End of Epoch 26 | Time 38.82s | Test Loss -15.996
+Fund new best model, dict saved
+Train Summary | End of Epoch 27 | Time 813.20s | Train Loss -18.068
+Valid Summary | End of Epoch 27 | Time 62.98s | Valid Loss -17.208
+Test Summary | End of Epoch 27 | Time 38.97s | Test Loss -15.804
+Fund new best model, dict saved
+Train Summary | End of Epoch 28 | Time 812.99s | Train Loss -18.156
+Valid Summary | End of Epoch 28 | Time 62.74s | Valid Loss -17.093
+Test Summary | End of Epoch 28 | Time 38.68s | Test Loss -15.791
+Train Summary | End of Epoch 29 | Time 813.21s | Train Loss -18.218
+Valid Summary | End of Epoch 29 | Time 62.76s | Valid Loss -17.131
+Test Summary | End of Epoch 29 | Time 38.63s | Test Loss -16.079
+Train Summary | End of Epoch 30 | Time 812.77s | Train Loss -18.288
+Valid Summary | End of Epoch 30 | Time 62.89s | Valid Loss -17.246
+Test Summary | End of Epoch 30 | Time 39.09s | Test Loss -15.575
+Fund new best model, dict saved
+Train Summary | End of Epoch 31 | Time 813.24s | Train Loss -18.363
+Valid Summary | End of Epoch 31 | Time 62.55s | Valid Loss -16.372
+Test Summary | End of Epoch 31 | Time 38.43s | Test Loss -15.847
+Train Summary | End of Epoch 32 | Time 812.31s | Train Loss -18.428
+Valid Summary | End of Epoch 32 | Time 63.25s | Valid Loss -17.360
+Test Summary | End of Epoch 32 | Time 38.70s | Test Loss -15.936
+Fund new best model, dict saved
+Train Summary | End of Epoch 33 | Time 812.89s | Train Loss -18.476
+Valid Summary | End of Epoch 33 | Time 62.54s | Valid Loss -17.510
+Test Summary | End of Epoch 33 | Time 38.51s | Test Loss -16.210
+Fund new best model, dict saved
+Train Summary | End of Epoch 34 | Time 812.46s | Train Loss -18.518
+Valid Summary | End of Epoch 34 | Time 63.26s | Valid Loss -17.510
+Test Summary | End of Epoch 34 | Time 39.20s | Test Loss -16.004
+Fund new best model, dict saved
+Train Summary | End of Epoch 35 | Time 813.37s | Train Loss -18.586
+Valid Summary | End of Epoch 35 | Time 63.13s | Valid Loss -17.287
+Test Summary | End of Epoch 35 | Time 38.75s | Test Loss -16.044
+Train Summary | End of Epoch 36 | Time 813.24s | Train Loss -18.644
+Valid Summary | End of Epoch 36 | Time 62.65s | Valid Loss -17.716
+Test Summary | End of Epoch 36 | Time 38.10s | Test Loss -16.221
+Fund new best model, dict saved
+Train Summary | End of Epoch 37 | Time 812.83s | Train Loss -18.700
+Valid Summary | End of Epoch 37 | Time 62.28s | Valid Loss -15.359
+Test Summary | End of Epoch 37 | Time 38.34s | Test Loss -15.573
+Train Summary | End of Epoch 38 | Time 813.20s | Train Loss -18.734
+Valid Summary | End of Epoch 38 | Time 62.85s | Valid Loss -17.653
+Test Summary | End of Epoch 38 | Time 38.57s | Test Loss -16.252
+Train Summary | End of Epoch 39 | Time 812.63s | Train Loss -18.778
+Valid Summary | End of Epoch 39 | Time 62.37s | Valid Loss -17.796
+Test Summary | End of Epoch 39 | Time 38.66s | Test Loss -16.310
+Fund new best model, dict saved
+Train Summary | End of Epoch 40 | Time 812.81s | Train Loss -18.837
+Valid Summary | End of Epoch 40 | Time 62.47s | Valid Loss -17.680
+Test Summary | End of Epoch 40 | Time 38.50s | Test Loss -16.250
+Train Summary | End of Epoch 41 | Time 813.08s | Train Loss -18.883
+Valid Summary | End of Epoch 41 | Time 62.47s | Valid Loss -16.569
+Test Summary | End of Epoch 41 | Time 38.17s | Test Loss -15.858
+Train Summary | End of Epoch 42 | Time 813.25s | Train Loss -18.922
+Valid Summary | End of Epoch 42 | Time 63.14s | Valid Loss -16.997
+Test Summary | End of Epoch 42 | Time 38.78s | Test Loss -15.927
+Train Summary | End of Epoch 43 | Time 814.01s | Train Loss -18.962
+Valid Summary | End of Epoch 43 | Time 62.87s | Valid Loss -16.562
+Test Summary | End of Epoch 43 | Time 38.42s | Test Loss -16.005
+Train Summary | End of Epoch 44 | Time 813.39s | Train Loss -18.992
+Valid Summary | End of Epoch 44 | Time 62.70s | Valid Loss -17.942
+Test Summary | End of Epoch 44 | Time 38.64s | Test Loss -16.475
+Fund new best model, dict saved
+Train Summary | End of Epoch 45 | Time 812.79s | Train Loss -19.024
+Valid Summary | End of Epoch 45 | Time 62.52s | Valid Loss -17.861
+Test Summary | End of Epoch 45 | Time 38.57s | Test Loss -16.471
+Train Summary | End of Epoch 46 | Time 813.03s | Train Loss -19.068
+Valid Summary | End of Epoch 46 | Time 62.68s | Valid Loss -17.831
+Test Summary | End of Epoch 46 | Time 38.32s | Test Loss -16.308
+Train Summary | End of Epoch 47 | Time 813.93s | Train Loss -19.106
+Valid Summary | End of Epoch 47 | Time 120.20s | Valid Loss -11.629
+Test Summary | End of Epoch 47 | Time 85.00s | Test Loss -15.160
+Train Summary | End of Epoch 48 | Time 4301.71s | Train Loss -19.142
+Valid Summary | End of Epoch 48 | Time 144.13s | Valid Loss -18.040
+Test Summary | End of Epoch 48 | Time 79.20s | Test Loss -16.315
+Fund new best model, dict saved
+Train Summary | End of Epoch 49 | Time 1939.32s | Train Loss -19.162
+Valid Summary | End of Epoch 49 | Time 126.16s | Valid Loss -18.018
+Test Summary | End of Epoch 49 | Time 75.46s | Test Loss -16.465
+Train Summary | End of Epoch 50 | Time 1939.72s | Train Loss -19.214
+Valid Summary | End of Epoch 50 | Time 126.05s | Valid Loss -18.083
+Test Summary | End of Epoch 50 | Time 75.15s | Test Loss -16.414
+Fund new best model, dict saved
+Train Summary | End of Epoch 51 | Time 1933.84s | Train Loss -19.243
+Valid Summary | End of Epoch 51 | Time 125.93s | Valid Loss -18.084
+Test Summary | End of Epoch 51 | Time 75.75s | Test Loss -16.608
+Fund new best model, dict saved
+Train Summary | End of Epoch 52 | Time 1939.58s | Train Loss -19.278
+Valid Summary | End of Epoch 52 | Time 126.17s | Valid Loss -17.739
+Test Summary | End of Epoch 52 | Time 75.92s | Test Loss -16.018
+Train Summary | End of Epoch 53 | Time 1939.34s | Train Loss -19.297
+Valid Summary | End of Epoch 53 | Time 126.20s | Valid Loss -18.189
+Test Summary | End of Epoch 53 | Time 75.89s | Test Loss -16.645
+Fund new best model, dict saved
+Train Summary | End of Epoch 54 | Time 1942.88s | Train Loss -19.345
+Valid Summary | End of Epoch 54 | Time 126.39s | Valid Loss -18.121
+Test Summary | End of Epoch 54 | Time 75.22s | Test Loss -16.255
+Train Summary | End of Epoch 55 | Time 1940.59s | Train Loss -19.363
+Valid Summary | End of Epoch 55 | Time 125.52s | Valid Loss -18.270
+Test Summary | End of Epoch 55 | Time 75.28s | Test Loss -16.622
+Fund new best model, dict saved
+Train Summary | End of Epoch 56 | Time 1935.52s | Train Loss -19.391
+Valid Summary | End of Epoch 56 | Time 126.14s | Valid Loss -17.902
+Test Summary | End of Epoch 56 | Time 75.28s | Test Loss -16.636
+Train Summary | End of Epoch 57 | Time 1947.40s | Train Loss -19.403
+Valid Summary | End of Epoch 57 | Time 141.21s | Valid Loss -18.258
+Test Summary | End of Epoch 57 | Time 81.67s | Test Loss -16.704
+Train Summary | End of Epoch 58 | Time 1937.52s | Train Loss -19.442
+Valid Summary | End of Epoch 58 | Time 125.90s | Valid Loss -18.183
+Test Summary | End of Epoch 58 | Time 76.09s | Test Loss -16.447
+Train Summary | End of Epoch 59 | Time 1938.74s | Train Loss -19.460
+Valid Summary | End of Epoch 59 | Time 125.72s | Valid Loss -18.221
+Test Summary | End of Epoch 59 | Time 75.65s | Test Loss -16.484
+Train Summary | End of Epoch 60 | Time 1938.40s | Train Loss -19.510
+Valid Summary | End of Epoch 60 | Time 126.54s | Valid Loss -18.143
+Test Summary | End of Epoch 60 | Time 76.00s | Test Loss -16.491
+reload weights and optimizer from last best checkpoint
+Learning rate adjusted to: 0.000500
+Train Summary | End of Epoch 61 | Time 1938.53s | Train Loss -19.653
+Valid Summary | End of Epoch 61 | Time 125.69s | Valid Loss -18.524
+Test Summary | End of Epoch 61 | Time 75.75s | Test Loss -16.811
+Fund new best model, dict saved
+Train Summary | End of Epoch 62 | Time 1935.76s | Train Loss -19.721
+Valid Summary | End of Epoch 62 | Time 125.88s | Valid Loss -18.546
+Test Summary | End of Epoch 62 | Time 75.65s | Test Loss -16.786
+Fund new best model, dict saved
+Train Summary | End of Epoch 63 | Time 1938.29s | Train Loss -19.752
+Valid Summary | End of Epoch 63 | Time 126.20s | Valid Loss -18.549
+Test Summary | End of Epoch 63 | Time 75.38s | Test Loss -16.896
+Fund new best model, dict saved
+Train Summary | End of Epoch 64 | Time 1938.66s | Train Loss -19.794
+Valid Summary | End of Epoch 64 | Time 125.45s | Valid Loss -18.592
+Test Summary | End of Epoch 64 | Time 75.49s | Test Loss -16.868
+Fund new best model, dict saved
+Train Summary | End of Epoch 65 | Time 1937.49s | Train Loss -19.819
+Valid Summary | End of Epoch 65 | Time 125.89s | Valid Loss -18.524
+Test Summary | End of Epoch 65 | Time 75.28s | Test Loss -16.894
+Train Summary | End of Epoch 66 | Time 1936.33s | Train Loss -19.840
+Valid Summary | End of Epoch 66 | Time 125.93s | Valid Loss -18.600
+Test Summary | End of Epoch 66 | Time 75.42s | Test Loss -16.922
+Fund new best model, dict saved
+Train Summary | End of Epoch 67 | Time 1938.10s | Train Loss -19.870
+Valid Summary | End of Epoch 67 | Time 125.96s | Valid Loss -18.605
+Test Summary | End of Epoch 67 | Time 75.25s | Test Loss -16.973
+Fund new best model, dict saved
+Train Summary | End of Epoch 68 | Time 1937.18s | Train Loss -19.883
+Valid Summary | End of Epoch 68 | Time 125.45s | Valid Loss -18.623
+Test Summary | End of Epoch 68 | Time 75.31s | Test Loss -17.031
+Fund new best model, dict saved
+Train Summary | End of Epoch 69 | Time 1934.23s | Train Loss -19.902
+Valid Summary | End of Epoch 69 | Time 125.81s | Valid Loss -18.522
+Test Summary | End of Epoch 69 | Time 75.71s | Test Loss -16.899
+Train Summary | End of Epoch 70 | Time 1937.90s | Train Loss -19.928
+Valid Summary | End of Epoch 70 | Time 125.90s | Valid Loss -18.610
+Test Summary | End of Epoch 70 | Time 75.16s | Test Loss -16.751
+Train Summary | End of Epoch 71 | Time 1939.63s | Train Loss -19.949
+Valid Summary | End of Epoch 71 | Time 126.40s | Valid Loss -18.579
+Test Summary | End of Epoch 71 | Time 75.48s | Test Loss -16.833
+Train Summary | End of Epoch 72 | Time 1939.05s | Train Loss -19.958
+Valid Summary | End of Epoch 72 | Time 125.71s | Valid Loss -18.654
+Test Summary | End of Epoch 72 | Time 75.39s | Test Loss -16.874
+Fund new best model, dict saved
+Train Summary | End of Epoch 73 | Time 1935.47s | Train Loss -19.978
+Valid Summary | End of Epoch 73 | Time 126.41s | Valid Loss -18.638
+Test Summary | End of Epoch 73 | Time 75.85s | Test Loss -16.873
+Train Summary | End of Epoch 74 | Time 1937.77s | Train Loss -19.988
+Valid Summary | End of Epoch 74 | Time 126.00s | Valid Loss -18.645
+Test Summary | End of Epoch 74 | Time 75.82s | Test Loss -16.738
+Train Summary | End of Epoch 75 | Time 1938.83s | Train Loss -20.012
+Valid Summary | End of Epoch 75 | Time 125.59s | Valid Loss -18.645
+Test Summary | End of Epoch 75 | Time 75.37s | Test Loss -16.910
+Train Summary | End of Epoch 76 | Time 1937.47s | Train Loss -20.013
+Valid Summary | End of Epoch 76 | Time 126.25s | Valid Loss -18.678
+Test Summary | End of Epoch 76 | Time 73.25s | Test Loss -17.020
+Fund new best model, dict saved
+Train Summary | End of Epoch 77 | Time 1934.53s | Train Loss -20.033
+Valid Summary | End of Epoch 77 | Time 125.89s | Valid Loss -18.653
+Test Summary | End of Epoch 77 | Time 75.59s | Test Loss -16.875
+Train Summary | End of Epoch 78 | Time 1939.31s | Train Loss -20.049
+Valid Summary | End of Epoch 78 | Time 125.67s | Valid Loss -18.647
+Test Summary | End of Epoch 78 | Time 75.40s | Test Loss -16.805
+Train Summary | End of Epoch 79 | Time 1939.14s | Train Loss -20.066
+Valid Summary | End of Epoch 79 | Time 125.73s | Valid Loss -18.706
+Test Summary | End of Epoch 79 | Time 75.69s | Test Loss -16.929
+Fund new best model, dict saved
+Train Summary | End of Epoch 80 | Time 1934.85s | Train Loss -20.078
+Valid Summary | End of Epoch 80 | Time 125.68s | Valid Loss -18.632
+Test Summary | End of Epoch 80 | Time 75.14s | Test Loss -16.703
+Train Summary | End of Epoch 81 | Time 1937.54s | Train Loss -20.091
+Valid Summary | End of Epoch 81 | Time 125.79s | Valid Loss -18.727
+Test Summary | End of Epoch 81 | Time 75.17s | Test Loss -16.949
+Fund new best model, dict saved
+Train Summary | End of Epoch 82 | Time 1940.97s | Train Loss -20.106
+Valid Summary | End of Epoch 82 | Time 125.99s | Valid Loss -18.659
+Test Summary | End of Epoch 82 | Time 75.62s | Test Loss -16.828
+Train Summary | End of Epoch 83 | Time 1940.60s | Train Loss -20.111
+Valid Summary | End of Epoch 83 | Time 125.86s | Valid Loss -18.713
+Test Summary | End of Epoch 83 | Time 75.38s | Test Loss -16.985
+Train Summary | End of Epoch 84 | Time 1934.57s | Train Loss -20.121
+Valid Summary | End of Epoch 84 | Time 126.02s | Valid Loss -18.723
+Test Summary | End of Epoch 84 | Time 75.73s | Test Loss -16.886
+Train Summary | End of Epoch 85 | Time 1939.74s | Train Loss -20.136
+Valid Summary | End of Epoch 85 | Time 138.39s | Valid Loss -18.691
+Test Summary | End of Epoch 85 | Time 80.10s | Test Loss -16.823
+Train Summary | End of Epoch 86 | Time 1937.70s | Train Loss -20.150
+Valid Summary | End of Epoch 86 | Time 125.96s | Valid Loss -18.710
+Test Summary | End of Epoch 86 | Time 75.33s | Test Loss -16.945
+reload weights and optimizer from last best checkpoint
+Learning rate adjusted to: 0.000250
+Train Summary | End of Epoch 87 | Time 1937.52s | Train Loss -20.204
+Valid Summary | End of Epoch 87 | Time 125.28s | Valid Loss -18.809
+Test Summary | End of Epoch 87 | Time 75.78s | Test Loss -16.972
+Fund new best model, dict saved
+Train Summary | End of Epoch 88 | Time 1938.42s | Train Loss -20.240
+Valid Summary | End of Epoch 88 | Time 125.41s | Valid Loss -18.818
+Test Summary | End of Epoch 88 | Time 75.98s | Test Loss -17.035
+Fund new best model, dict saved
+Train Summary | End of Epoch 89 | Time 1939.54s | Train Loss -20.249
+Valid Summary | End of Epoch 89 | Time 126.38s | Valid Loss -18.812
+Test Summary | End of Epoch 89 | Time 76.28s | Test Loss -16.910
+Train Summary | End of Epoch 90 | Time 1945.55s | Train Loss -20.267
+Valid Summary | End of Epoch 90 | Time 126.46s | Valid Loss -18.806
+Test Summary | End of Epoch 90 | Time 75.42s | Test Loss -16.897
+Train Summary | End of Epoch 91 | Time 1949.44s | Train Loss -20.279
+Valid Summary | End of Epoch 91 | Time 124.58s | Valid Loss -18.810
+Test Summary | End of Epoch 91 | Time 75.95s | Test Loss -16.901
+Train Summary | End of Epoch 92 | Time 1941.30s | Train Loss -20.293
+Valid Summary | End of Epoch 92 | Time 124.42s | Valid Loss -18.809
+Test Summary | End of Epoch 92 | Time 75.64s | Test Loss -16.954
+Train Summary | End of Epoch 93 | Time 1941.58s | Train Loss -20.297
+Valid Summary | End of Epoch 93 | Time 125.85s | Valid Loss -18.787
+Test Summary | End of Epoch 93 | Time 75.74s | Test Loss -16.888
+reload weights and optimizer from last best checkpoint
+Learning rate adjusted to: 0.000125
+Train Summary | End of Epoch 94 | Time 1942.52s | Train Loss -20.291
+Valid Summary | End of Epoch 94 | Time 273.14s | Valid Loss -18.839
+Test Summary | End of Epoch 94 | Time 162.06s | Test Loss -17.012
+Fund new best model, dict saved
+Train Summary | End of Epoch 95 | Time 1985.76s | Train Loss -20.312
+Valid Summary | End of Epoch 95 | Time 228.44s | Valid Loss -18.840
+Test Summary | End of Epoch 95 | Time 155.77s | Test Loss -17.009
+Fund new best model, dict saved
+Train Summary | End of Epoch 96 | Time 2972.21s | Train Loss -20.316
+Valid Summary | End of Epoch 96 | Time 210.59s | Valid Loss -9.387
+Start evaluation
+Avg SISNR:i tensor([17.1080], device='cuda:0')
+Avg SNRi: 17.45552202765349
+Avg STOIi: 0.21840867744715423

checkpoints/log_wsj0-2mix_speech_SpEx-plus_2spk/tensorboard/events.out.tfevents.1727856848.dsw-106518-965b74ddc-cdclg.3549648.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1932d6a2ebf5f030cf4bc8a3367376ccb0346d292c57480ab66f1c59043454f4
+size 14148